<commit_before>"""Flask Backend Server for managing audio content.

Please see README.md for instructions.

Starting Locally
----------------
You have two options:

  $ python main.py --local --debug

Or, to use GCP backend by default:

  $ dev_appserver.py .


Endpoints
---------
  - /audio/upload : POST
  - /audio/<uri> : GET
  - /annotation/submit : POST
  - /annotation/taxonomy : GET
"""

import argparse
import datetime
from flask import Flask, request, Response
from flask import send_file
import io
import json
import logging
import requests
import os

import pybackend.database
import pybackend.mime
import pybackend.storage
import pybackend.utils


logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)

# Set the cloud backend
# TODO: This should be controlled by `app.yaml`, right?
CLOUD_CONFIG = os.path.join(os.path.dirname(__file__), 'gcloud_config.json')
app.config['cloud'] = json.load(open(CLOUD_CONFIG))

SOURCE = "https://cosmir.github.io/open-mic/"


@app.route('/')
def hello():
    return 'oh hai'


@app.route('/audio/upload', methods=['POST'])
def audio_upload():
    """
    To POST files to this endpoint:

    $ curl -F "audio=@some_file.mp3" localhost:8080/audio/upload

    TODOs:
      - Store user data (who uploaded this? IP address?)
      -
    """

    data = json.dumps(dict(message='Invalid request'))
    status = 405
    if request.method == 'POST':
        audio_data = request.files['audio']
        bytestring = audio_data.stream.read()

        # Copy to cloud storage
        store = pybackend.storage.Storage(
            project_id=app.config['cloud']['project_id'],
            **app.config['cloud']['storage'])

        uri = str(pybackend.utils.uuid(bytestring))
        fext = os.path.splitext(audio_data.filename)[-1]
        filepath = "{}{}".format(uri, fext)
        store.upload(bytestring, filepath)

        # Index in datastore
        # Keep things like extension, storage platform, mimetype, etc.
        dbase = pybackend.database.Database(
            project_id=app.config['cloud']['project_id'],
            **app.config['cloud']['database'])
        record = dict(filepath=filepath,
                      created=str(datetime.datetime.now()))
        dbase.put(uri, record)
        record.update(
            uri=uri,
            message="Received {} bytes of data.".format(len(bytestring)))
        status = 200
        data = json.dumps(record)

    resp = Response(data, status=status,
                    mimetype=pybackend.mime.MIMETYPES['json'])
    resp.headers['Link'] = SOURCE
    return resp


@app.route('/audio/<uri>', methods=['GET'])
def audio_download(uri):
    """
    To GET responses from this endpoint:

    $ curl -XGET localhost:8080/audio/bbdde322-c604-4753-b828-9fe8addf17b9
    """

    if request.method == 'GET':
        dbase = pybackend.database.Database(
            project_id=app.config['cloud']['project_id'],
            **app.config['cloud']['database'])

        entity = dbase.get(uri)
        if entity is None:
            msg = "Resource not found: {}".format(uri)
            app.logger.info(msg)
            resp = Response(
                json.dumps(dict(message=msg)),
                status=404)

        else:
            store = pybackend.storage.Storage(
                project_id=app.config['cloud']['project_id'],
                **app.config['cloud']['storage'])

            data = store.download(entity['filepath'])
            app.logger.debug("Returning {} bytes".format(len(data)))

            resp = send_file(
                io.BytesIO(data),
                attachment_filename=entity['filepath'],
                mimetype=pybackend.mime.mimetype_for_file(entity['filepath']))

    else:
        # Flask does this for us.
        resp = Response(
            json.dumps(dict(message="METHOD NOT ALLOWED")),
            status=405)

    resp.headers['Link'] = SOURCE
    return resp


@app.route('/annotation/submit', methods=['POST'])
def annotation_submit():
    """
    To POST data to this endpoint:

    $ curl -H "Content-type: application/json" \
        -X POST localhost:8080/annotation/submit \
        -d '{"message":"Hello Data"}'
    """

    data = json.dumps(dict(message='METHOD NOT ALLOWED'))
    status = 400

    conds = [request.method == 'POST',
             request.headers['Content-Type'] == 'application/json']
    if all(conds):
        app.logger.info("Received Annotation:\n{}"
                        .format(json.dumps(request.json, indent=2)))
        # obj = json.loads(request.data)
        data = json.dumps(dict(message='Success!'))
        status = 200

    resp = Response(
        data, status=status, mimetype=pybackend.mime.MIMETYPES['json'])
    resp.headers['Link'] = SOURCE
    return resp


@app.route('/annotation/taxonomy', methods=['GET'])
def annotation_taxonomy():
    """
    To fetch data at this endpoint:

    $ curl -X GET localhost:8080/annotation/taxonomy

    TODO: Clean this up per @alastair's feedback.
    """
    data = json.dumps(dict(message='Resource not found'))
    status = 404

    tax_url = ("https://raw.githubusercontent.com/marl/jams/master/jams/"
               "schemata/namespaces/tag/medleydb_instruments.json")
    res = requests.get(tax_url)
    if res.text:
        data = json.loads(res.text)
        status = 200

    resp = Response(data, status=status)<commit_msg>Cleaned up request method filtering, which Flask does for us.<commit_after>"""Flask Backend Server for managing audio content.

Please see README.md for instructions.

Starting Locally
----------------
You have two options:

  $ python main.py --local --debug

Or, to use GCP backend by default:

  $ dev_appserver.py .


Endpoints
---------
  - /audio/upload : POST
  - /audio/<uri> : GET
  - /annotation/submit : POST
  - /annotation/taxonomy : GET
"""

import argparse
import datetime
from flask import Flask, request, Response
from flask import send_file
import io
import json
import logging
import requests
import os

import pybackend.database
import pybackend.mime
import pybackend.storage
import pybackend.utils


logging.basicConfig(level=logging.DEBUG)
app = Flask(__name__)

# Set the cloud backend
# TODO: This should be controlled by `app.yaml`, right?
CLOUD_CONFIG = os.path.join(os.path.dirname(__file__), 'gcloud_config.json')
app.config['cloud'] = json.load(open(CLOUD_CONFIG))

SOURCE = "https://cosmir.github.io/open-mic/"


@app.route('/')
def hello():
    return 'oh hai'


@app.route('/audio/upload', methods=['POST'])
def audio_upload():
    """
    To POST files to this endpoint:

    $ curl -F "audio=@some_file.mp3" localhost:8080/audio/upload

    TODOs:
      - Store user data (who uploaded this? IP address?)
      -
    """
    audio_data = request.files['audio']
    bytestring = audio_data.stream.read()

    # Copy to cloud storage
    store = pybackend.storage.Storage(
        project_id=app.config['cloud']['project_id'],
        **app.config['cloud']['storage'])

    uri = str(pybackend.utils.uuid(bytestring))
    fext = os.path.splitext(audio_data.filename)[-1]
    filepath = "{}{}".format(uri, fext)
    store.upload(bytestring, filepath)

    # Index in datastore
    # Keep things like extension, storage platform, mimetype, etc.
    dbase = pybackend.database.Database(
        project_id=app.config['cloud']['project_id'],
        **app.config['cloud']['database'])
    record = dict(filepath=filepath,
                  created=str(datetime.datetime.now()))
    dbase.put(uri, record)
    record.update(
        uri=uri,
        message="Received {} bytes of data.".format(len(bytestring)))

    resp = Response(json.dumps(record), status=200,
                    mimetype=pybackend.mime.MIMETYPES['json'])
    resp.headers['Link'] = SOURCE
    return resp


@app.route('/audio/<uri>', methods=['GET'])
def audio_download(uri):
    """
    To GET responses from this endpoint:

    $ curl -XGET localhost:8080/audio/bbdde322-c604-4753-b828-9fe8addf17b9
    """

    dbase = pybackend.database.Database(
        project_id=app.config['cloud']['project_id'],
        **app.config['cloud']['database'])

    entity = dbase.get(uri)
    if entity is None:
        msg = "Resource not found: {}".format(uri)
        app.logger.info(msg)
        resp = Response(
            json.dumps(dict(message=msg)),
            status=404)

    else:
        store = pybackend.storage.Storage(
            project_id=app.config['cloud']['project_id'],
            **app.config['cloud']['storage'])

        data = store.download(entity['filepath'])
        app.logger.debug("Returning {} bytes".format(len(data)))

        resp = send_file(
            io.BytesIO(data),
            attachment_filename=entity['filepath'],
            mimetype=pybackend.mime.mimetype_for_file(entity['filepath']))

    resp.headers['Link'] = SOURCE
    return resp


@app.route('/annotation/submit', methods=['POST'])
def annotation_submit():
    """
    To POST data to this endpoint:

    $ curl -H "Content-type: application/json" \
        -X POST localhost:8080/annotation/submit \
        -d '{"message":"Hello Data"}'
    """

    if request.headers['Content-Type'] == 'application/json':
        app.logger.info("Received Annotation:\n{}"
                        .format(json.dumps(request.json, indent=2)))
        # obj = json.loads(request.data)
        data = json.dumps(dict(message='Success!'))
        status = 200

    else:
        status = 400
        data = json.dumps(dict(message='Invalid Content-Type; '
                                       'only accepts application/json'))

    resp = Response(
        data, status=status, mimetype=pybackend.mime.MIMETYPES['json'])
    resp.headers['Link'] = SOURCE
    return resp


@app.route('/annotation/taxonomy', methods=['GET'])
def annotation_taxonomy():
    """
    To fetch data at this endpoint:

    $ curl -X GET localhost:8080/annotation/taxonomy

    TODO: Clean this up per @alastair's feedback.
    """
    data = json.dumps(dict(message='Resource not found'))
    status = 404

    tax_url = ("https://raw.githubusercontent.com/marl/jams/master/jams/"
               "schemata/namespaces/tag/medleydb_instruments.json")
    res = requests.get(tax_url)
    if res.text:
        data = json.loads(res.text)
        status = 200

    resp = Response(data, status=status)
{'fraction_non_alphanumeric': 0.1423601043780806, 'fraction_numerical': 0.011210979027737508, 'mean_word_length': 2.674715909090909, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2811608', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3077, 'n_words': 676}