<commit_before>from os import getenv
import json

from flask import Flask, request, jsonify, g
from flask_featureflags import FeatureFlag
from backdrop import statsd
from backdrop.core.data_set import DataSet
from backdrop.core.flaskutils import DataSetConverter
from backdrop.core.repository import (DataSetConfigRepository,
                                      UserConfigRepository)

from ..core.errors import ParseError, ValidationError
from ..core import database, log_handler, cache_control

from .validation import auth_header_is_valid, extract_bearer_token


GOVUK_ENV = getenv("GOVUK_ENV", "development")

app = Flask("backdrop.write.api")

feature_flags = FeatureFlag(app)

# Configuration
app.config.from_object(
    "backdrop.write.config.{}".format(GOVUK_ENV))

db = database.Database(
    app.config['MONGO_HOSTS'],
    app.config['MONGO_PORT'],
    app.config['DATABASE_NAME']
)

data_set_repository = DataSetConfigRepository(
    app.config['STAGECRAFT_URL'],
    app.config['STAGECRAFT_DATA_SET_QUERY_TOKEN'])

user_repository = UserConfigRepository(db)

log_handler.set_up_logging(app, GOVUK_ENV)

app.url_map.converters["data_set"] = DataSetConverter


@app.errorhandler(500)
@app.errorhandler(405)
@app.errorhandler(404)
def exception_handler(e):
    app.logger.exception(e)

    data_set_name = getattr(g, 'data_set_name', request.path)
    statsd.incr("write.error", data_set=data_set_name)

    code = getattr(e, 'code', 500)
    name = getattr(e, 'name', "Internal Error")

    return jsonify(status='error', message=name), code


@app.route('/_status', methods=['GET'])
@cache_control.nocache
def health_check():
    if db.alive():
        return jsonify(status='ok', message='database seems fine')
    else:
        return jsonify(status='error',
                       message='cannot connect to database'), 500


@app.route('/data/<data_group>/<data_type>', methods=['POST'])
@cache_control.nocache
def write_by_group(data_group, data_type):
    """
    Write by group/type
    e.g. POST https://BACKDROP/data/my-transaction-name/volumetrics
    """
    data_set_config = data_set_repository.get_data_set_for_query(
        data_group,
        data_type)
    return _write_to_data_set(data_set_config)


@app.route('/<data_set:data_set_name>', methods=['POST'])
@cache_control.nocache
def post_to_data_set(data_set_name):
    app.logger.warning("Deprecated use of write API by name: {}".format(
        data_set_name))
    data_set_config = data_set_repository.retrieve(name=data_set_name)
    return _write_to_data_set(
        data_set_config,
        ok_message="Deprecation Warning: accessing by data-set name is "
                   "deprecated, Please use the /data-group/data-type form.")


@app.route('/data-sets/<dataset_name>', methods=['POST'])
@cache_control.nocache
def create_collection_for_dataset(dataset_name):
    if not _allow_create_collection(request.headers.get('Authorization')):
        return jsonify(status='error',
                       message="Forbidden: invalid or no token given."), 403

    if db.collection_exists(dataset_name):
        return jsonify(status='error',
                       message='Collection exists with that name.'), 400

    try:
        data = json.loads(request.data)
    except ValueError as e:
        return jsonify(status='error', message=repr(e)), 400
    else:
        capped_size = data.get('capped_size', None)

    if capped_size is None or not isinstance(capped_size, int):
        return jsonify(
            status='error',
            message="You must specify an int capped_size of 0 or more"), 400

    if capped_size == 0:
        db.create_uncapped_collection(dataset_name)
    else:
        db.create_capped_collection(dataset_name, capped_size)

    return jsonify(status='ok', message='Created "{}"'.format(dataset_name))


@app.route('/data-sets/<dataset_name>', methods=['DELETE'])
@cache_control.nocache
def delete_collection_by_dataset_name(dataset_name):
    if not _allow_create_collection(request.headers.get('Authorization')):
        return jsonify(status='error',
                       message="Forbidden: invalid or no token given."), 403

    if not db.collection_exists(dataset_name):
        return jsonify(
            status='error',
            message='No collection exists with name "{}"'.format(dataset_name)
        ), 404

    db.delete_collection(dataset_name)

    return jsonify(status='ok', message='Deleted {}'.format(dataset_name))


def _allow_modify_collection(auth_header):
    token = extract_bearer_token(auth_header)
    if token == app.config['STAGECRAFT_COLLECTION_ENDPOINT_TOKEN']:
        return True

    app.logger.info("Bad token for create collection: '{}'".format(token))
    return False


def _allow_delete_collection(auth_header):
    return _allow_modify_collection(auth_header)


def _allow_create_collection(auth_header):
    return _allow_modify_collection(auth_header)


def _write_to_data_set(data_set_config, ok_message=None):
    if data_set_config is None:
        return jsonify(status="error",
                       message='Could not find data_set_config'), 404

    g.data_set_name = data_set_config.name

    try:
        auth_header = request.headers['Authorization']
    except KeyError:
        return jsonify(status='error',
                       message='Authorization header missing.'), 403

    if not auth_header_is_valid(data_set_config, auth_header):
        statsd.incr("write_api.bad_token", data_set=g.data_set_name)
        return jsonify(status='error', message='Forbidden'), 403

    try:
        data = listify_json(request.json)

        data_set = DataSet(db, data_set_config)
        data_set.parse_and_store(data)
        if ok_message:
            return jsonify(status='ok', message=ok_message)
        else:
            return jsonify(status='ok')

    except (ParseError, ValidationError) as e:
        return jsonify(status="error", message=str(e)), 400

<commit_msg>Use abort() for error handling<commit_after>from os import getenv
import json

from flask import abort, Flask, g, jsonify, request
from flask_featureflags import FeatureFlag
from backdrop import statsd
from backdrop.core.data_set import DataSet
from backdrop.core.flaskutils import DataSetConverter
from backdrop.core.repository import (DataSetConfigRepository,
                                      UserConfigRepository)

from ..core.errors import ParseError, ValidationError
from ..core import database, log_handler, cache_control

from .validation import auth_header_is_valid, extract_bearer_token


GOVUK_ENV = getenv("GOVUK_ENV", "development")

app = Flask("backdrop.write.api")

feature_flags = FeatureFlag(app)

# Configuration
app.config.from_object(
    "backdrop.write.config.{}".format(GOVUK_ENV))

db = database.Database(
    app.config['MONGO_HOSTS'],
    app.config['MONGO_PORT'],
    app.config['DATABASE_NAME']
)

data_set_repository = DataSetConfigRepository(
    app.config['STAGECRAFT_URL'],
    app.config['STAGECRAFT_DATA_SET_QUERY_TOKEN'])

user_repository = UserConfigRepository(db)

log_handler.set_up_logging(app, GOVUK_ENV)

app.url_map.converters["data_set"] = DataSetConverter


@app.errorhandler(400)
@app.errorhandler(403)
@app.errorhandler(404)
@app.errorhandler(405)
@app.errorhandler(500)
def exception_handler(e):
    app.logger.exception(e)

    data_set_name = getattr(g, 'data_set_name', request.path)
    statsd.incr("write.error", data_set=data_set_name)

    code = getattr(e, 'code', 500)
    description = getattr(e, 'description', "Unknown Error")

    return jsonify(status='error', message=description), code


@app.route('/_status', methods=['GET'])
@cache_control.nocache
def health_check():
    if db.alive():
        return jsonify(status='ok', message='database seems fine')
    else:
        abort(500, 'cannot connect to database')


@app.route('/data/<data_group>/<data_type>', methods=['POST'])
@cache_control.nocache
def write_by_group(data_group, data_type):
    """
    Write by group/type
    e.g. POST https://BACKDROP/data/my-transaction-name/volumetrics
    """
    data_set_config = data_set_repository.get_data_set_for_query(
        data_group,
        data_type)
    return _write_to_data_set(data_set_config)


@app.route('/<data_set:data_set_name>', methods=['POST'])
@cache_control.nocache
def post_to_data_set(data_set_name):
    app.logger.warning("Deprecated use of write API by name: {}".format(
        data_set_name))
    data_set_config = data_set_repository.retrieve(name=data_set_name)
    return _write_to_data_set(
        data_set_config,
        ok_message="Deprecation Warning: accessing by data-set name is "
                   "deprecated, Please use the /data-group/data-type form.")


@app.route('/data-sets/<dataset_name>', methods=['POST'])
@cache_control.nocache
def create_collection_for_dataset(dataset_name):
    if not _allow_create_collection(request.headers.get('Authorization')):
        abort(403, 'Forbidden: invalid or no token given.')

    if db.collection_exists(dataset_name):
        abort(400, 'Collection exists with that name.')

    try:
        data = json.loads(request.data)
    except ValueError as e:
        abort(400, repr(e))
    else:
        capped_size = data.get('capped_size', None)

    if capped_size is None or not isinstance(capped_size, int):
        abort(400, 'You must specify an int capped_size of 0 or more')

    if capped_size == 0:
        db.create_uncapped_collection(dataset_name)
    else:
        db.create_capped_collection(dataset_name, capped_size)

    return jsonify(status='ok', message='Created "{}"'.format(dataset_name))


@app.route('/data-sets/<dataset_name>', methods=['DELETE'])
@cache_control.nocache
def delete_collection_by_dataset_name(dataset_name):
    if not _allow_create_collection(request.headers.get('Authorization')):
        abort(403, 'Forbidden: invalid or no token given.')

    if not db.collection_exists(dataset_name):
        abort(404, 'No collection exists with name "{}"'.format(dataset_name))

    db.delete_collection(dataset_name)

    return jsonify(status='ok', message='Deleted {}'.format(dataset_name))


def _allow_modify_collection(auth_header):
    token = extract_bearer_token(auth_header)
    if token == app.config['STAGECRAFT_COLLECTION_ENDPOINT_TOKEN']:
        return True

    app.logger.info("Bad token for create collection: '{}'".format(token))
    return False


def _allow_delete_collection(auth_header):
    return _allow_modify_collection(auth_header)


def _allow_create_collection(auth_header):
    return _allow_modify_collection(auth_header)


def _write_to_data_set(data_set_config, ok_message=None):
    if data_set_config is None:
        abort(404, 'Could not find data_set_config')

    g.data_set_name = data_set_config.name

    try:
        auth_header = request.headers['Authorization']
    except KeyError:
        abort(403, 'Authorization header missing.')

    if not auth_header_is_valid(data_set_config, auth_header):
        statsd.incr("write_api.bad_token", data_set=g.data_set_name)
        abort(403, 'Forbidden')

    try:
        data = listify_json(request.json)

        data_set = DataSet(db, data_set_config)
        data_set.parse_and_store(data)
        if ok_message:
            return jsonify(status='ok', message=ok_message)
        else:
            return jsonify(status='ok')

    except (ParseError, ValidationError) as e:
        abort(400, str(e))


{'fraction_non_alphanumeric': 0.11054837582954942, 'fraction_numerical': 0.008732099196646874, 'mean_word_length': 3.960155911650065, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '770779', 'n_tokens_mistral': 3708, 'n_tokens_neox': 3561, 'n_words': 716}