<commit_before>#!/usr/bin/env python

from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()

from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request


import json
import os

from flask import Flask
from flask import request
from flask import make_response

# Flask app should start in global layout
app = Flask(__name__)


@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = processRequest(req)

    res = json.dumps(res, indent=4)
    print("RES\n" + type(res))
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r


def processRequest(req):
    if req.get("result").get("action") != "DuckDuckGoInstantAnswer":
        return {}
    baseurl = "http://api.duckduckgo.com/?"
    yql_query = makeYqlQuery(req)
    print ("YQL_QUERY\n" + yql_query)
    if yql_query is None:
        return {}
    yql_url = baseurl + urlencode({'q': yql_query}) + "&format=json"
    print("yql_url:" + yql_url)
    result = urlopen(yql_url).read().decode("utf8")
    print (result)
    data = json.loads(result)
    print (data)
    res = makeWebhookResult(data)
    return res


def makeYqlQuery(req):
    result = req.get("result")
    query = result.get("resolvedQuery")
    if query is None:
        return None

    return query


def makeWebhookResult(data):
    answer = data.get('Abstract')
    if answer is None:
        return {}

    print(json.dumps(answer, indent=4))

    speech = answer

    print("Response:")
    print(speech)

    return {
        "speech": speech,
        "displayText": speech,
        # "data": data,
        # "contextOut": [],
        "source": "apiai-duckduckgo-webhook"
    }


if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print("Starting app on port %d" % port)

    app.run(debug=False, port=port, host='0.0.0.0')
<commit_msg>edits<commit_after>#!/usr/bin/env python

from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()

from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request


import json
import os

from flask import Flask
from flask import request
from flask import make_response

# Flask app should start in global layout
app = Flask(__name__)


@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = processRequest(req)

    res = json.dumps(res, indent=4)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r


def processRequest(req):
    if req.get("result").get("action") != "DuckDuckGoInstantAnswer":
        return {}
    baseurl = "http://api.duckduckgo.com/?"
    yql_query = makeYqlQuery(req)
    print ("YQL_QUERY\n" + yql_query)
    if yql_query is None:
        return {}
    yql_url = baseurl + urlencode({'q': yql_query}) + "&format=json"
    print("yql_url:" + yql_url)
    result = urlopen(yql_url).read().decode("utf8")
    print (result)
    data = json.loads(result)
    print (data)
    res = makeWebhookResult(data)
    return res


def makeYqlQuery(req):
    result = req.get("result")
    query = result.get("resolvedQuery")
    if query is None:
        return None

    return query


def makeWebhookResult(data):
    answer = data.get('Abstract')
    if answer is None:
        return {}

    print(json.dumps(answer, indent=4))

    speech = answer

    print("Response:")
    print(speech)

    return {
        "speech": speech,
        "displayText": speech,
        # "data": data,
        # "contextOut": [],
        "source": "apiai-duckduckgo-webhook"
    }


if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print("Starting app on port %d" % port)

    app.run(debug=False, port=port, host='0.0.0.0')

{'fraction_non_alphanumeric': 0.13386414530977855, 'fraction_numerical': 0.005971634735008708, 'mean_word_length': 3.244984160506864, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '24833', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1338, 'n_words': 318}