<commit_before>from flask import Flask, request, Response
import json
import sys
import os
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))

from api import process_docs
import search

app = Flask(__name__)


@app.route('/process_raw', methods=['GET', 'POST'])
def process_raw():
    if request.method == 'POST':
        doc = request.form.get('doc')
        source = request.form.get('source')
        doc_id = request.form.get('doc_id')
        filetype = request.form.get('filetype')
    else:
        doc = request.args.get('doc')
        source = request.args.get('source')
        doc_id = request.args.get('doc_id')
        filetype = request.args.get('filetype')

    return Response(process_docs.process_raw(doc, source, doc_id, filetype))


@app.route('/process', methods=['GET', 'POST'])
def process():
    if request.method == 'POST':
        doc = json.loads(request.form.get('doc'))
        timestamp = request.form.get('timestamp')
    else:
        doc = json.loads(request.args.get('doc'))
        timestamp = request.args.get('timestamp')
    processed_doc = process_docs.process(doc, timestamp)
    search.update('scrapi', doc, 'article', doc['id'])
    return processed_doc


@app.route('/search', methods=['GET'])
def search_search():
    query = request.args.get('q')
    start = request.args.get('from')
    size = request.args.get('size')
    return Response(json.dumps(search.search(query, start, size), indent=4, sort_keys=True), mimetype='application/json')

if __name__ == '__main__':
    app.run(
        host="0.0.0.0",
        port=1337,
        debug=True
    )
<commit_msg>GET and POST requests can now process lists<commit_after>from flask import Flask, request, Response
import json
import sys
import os
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))

from api import process_docs
import search

app = Flask(__name__)


@app.route('/process_raw', methods=['GET', 'POST'])
def process_raw():
    if request.method == 'POST':
        docs = request.form['doc']
        doc_list_item = docs.split("ASDFJKL")
        doc_ids = request.form['doc_id']
        doc_ids_item = doc_ids.split("ASDFJKL")
        for x in range(0,len(doc_list_item)):
            doc = doc_list_item[x]
            source = request.form.get('source')
            doc_id = doc_ids_item[x]
            filetype = request.form.get('filetype')

            Response(process_docs.process_raw(doc, source, doc_id, filetype))

    else:
        docs = request.args['doc']
        doc_list_item = docs.split("ASDFJKL")
        doc_ids = request.args['doc_id']
        doc_ids_item = doc_ids.split("ASDFJKL")
        for x in range(0,len(doc_list_item)):
            doc = doc_list_item[x]
            source = request.args.get('source')
            doc_id = doc_ids_item[x]
            filetype = request.args.get('filetype')

            Response(process_docs.process_raw(doc, source, doc_id, filetype))
    return "Processed!"


@app.route('/process', methods=['GET', 'POST'])
def process():
    if request.method == 'POST':
        doc = json.loads(request.form.get('doc'))
        timestamp = request.form.get('timestamp')
    else:
        doc = json.loads(request.args.get('doc'))
        timestamp = request.args.get('timestamp')
    processed_doc = process_docs.process(doc, timestamp)
    search.update('scrapi', doc, 'article', doc['id'])
    return processed_doc


@app.route('/search', methods=['GET'])
def search_search():
    query = request.args.get('q')
    start = request.args.get('from')
    size = request.args.get('size')
    return Response(json.dumps(search.search('scrapi', query, start, size), indent=4, sort_keys=True), mimetype='application/json')

if __name__ == '__main__':
    app.run(
        host="0.0.0.0",
        port=1337,
        debug=True
    )

{'fraction_non_alphanumeric': 0.15444015444015444, 'fraction_numerical': 0.005662805662805663, 'mean_word_length': 3.1208907741251326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2682190', 'n_tokens_mistral': 1282, 'n_tokens_neox': 1256, 'n_words': 245}