<commit_before>from collections import defaultdict
import datetime
import os
import requests
from urllib.parse import parse_qsl, urlparse

from flask import (
    abort,
    redirect,
    render_template,
    request,
    send_from_directory,
    url_for
)
from flask_login import (
    current_user,
    login_required,
    login_user,
    logout_user,
)
from furl import furl

from app import app, csrf, login_manager

from formendpoint.forms import EndpointForm
from formendpoint.models import (
    Destination,
    Endpoint,
    EndpointDestination,
    Gmail,
    GoogleDestinationMixin,
    GoogleSheet,
    Organization,
    PersonalDestinationMixin,
    Submission,
    User,
    db,
)

DEMO_URL = 'https://docs.google.com/spreadsheets/d/1QWeHPvZW4atIZxobdVXr3IYl8u4EnV99Dm_K4yGfo_8/'
GOOGLE_PICKER_API_KEY = os.environ['GOOGLE_PICKER_API_KEY']
GOOGLE_CLIENT_ID = os.environ['GOOGLE_CLIENT_ID']
GOOGLE_APP_ID = os.environ['GOOGLE_APP_ID']


def handle_submission(request, endpoint):
    """
    Data priority:
    1. Form data
    2. JSON body data
    3. request query parameters
    4. referrer query parameters

    Referrer parameters are not necessarily meaningful, but better to store in case.
    Within form or query params, you can have overlapping names to send lists, but
    between form, body, or query params, names overwrite.

    Special fields: email, referrer, next, names ending in ~
    - email: used for email notifications
    - referrer: overriding the referrer header value
    - next: url for redirecting
    - <name>~: for error messages
    """

    # Referrer
    data = defaultdict(list)
    for k, v in parse_qsl(urlparse(request.headers.get('REFERER')).query, keep_blank_values=True):
        data[k].append(v)

    # Request
    for arg in request.args:
        data[arg] = request.args.getlist(arg) or data[arg]

    # Body
    data.update(request.get_json() or {})

    # Form
    for field in request.form:
        data[field] = request.form.getlist(field) or data[field]

    # Flatten single item lists
    for k in data:
        if isinstance(data[k], list) and len(data[k]) == 1:
            data[k] = data[k][0]

    # remove `redirect`
    try:
        del data['redirect']
    except KeyError:
        pass

    ip_address = request.remote_addr if request.remote_addr != '127.0.0.1' \
        else request.headers.get('X-Forwarded-For')

    submission = Submission(
        data=dict(data),
        endpoint_id=endpoint.id,
        referrer=request.headers.get('REFERER'),
        user_agent=request.args.get('USER-AGENT'),
        ip_address=ip_address,
    )
    db.session.add(submission)
    db.session.commit()

    from formendpoint.tasks import process_submission
    process_submission.delay(submission.id)

    if 'redirect' in request.args:
        return redirect(request.args.get('redirect'))
    else:
        return redirect(url_for('endpoint', endpoint_id=endpoint.id, _external=True))


@login_manager.user_loader
def load_user(user_id):
    return User.query.get(user_id)


login_manager.login_view = "app.login"


@app.route('/')
def index():
    if current_user.is_authenticated:
        endpoint = Endpoint.query.filter_by(organization=current_user.organization).first()
        if endpoint:
            return redirect(url_for('endpoint', endpoint_id=endpoint.id))
        return redirect(url_for('create_endpoint'))

    else:
        url = furl(url_for('endpoint', endpoint_id=1, _external=True))
        url.args['destination'] = DEMO_URL
        form = render_template('form.html', url=url.url, input='<input type="email" name="email">')
        return render_template('welcome.html', form=form, demo_url=DEMO_URL)


@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'), 'favicon.ico',<commit_msg>fixing next param<commit_after>from collections import defaultdict
import datetime
import os
import requests
from urllib.parse import parse_qsl, urlparse

from flask import (
    abort,
    redirect,
    render_template,
    request,
    send_from_directory,
    url_for
)
from flask_login import (
    current_user,
    login_required,
    login_user,
    logout_user,
)
from furl import furl

from app import app, csrf, login_manager

from formendpoint.forms import EndpointForm
from formendpoint.models import (
    Destination,
    Endpoint,
    EndpointDestination,
    Gmail,
    GoogleDestinationMixin,
    GoogleSheet,
    Organization,
    PersonalDestinationMixin,
    Submission,
    User,
    db,
)

DEMO_URL = 'https://docs.google.com/spreadsheets/d/1QWeHPvZW4atIZxobdVXr3IYl8u4EnV99Dm_K4yGfo_8/'
GOOGLE_PICKER_API_KEY = os.environ['GOOGLE_PICKER_API_KEY']
GOOGLE_CLIENT_ID = os.environ['GOOGLE_CLIENT_ID']
GOOGLE_APP_ID = os.environ['GOOGLE_APP_ID']


def handle_submission(request, endpoint):
    """
    Data priority:
    1. Form data
    2. JSON body data
    3. request query parameters
    4. referrer query parameters

    Referrer parameters are not necessarily meaningful, but better to store in case.
    Within form or query params, you can have overlapping names to send lists, but
    between form, body, or query params, names overwrite.

    Special fields: email, referrer, next, names ending in ~
    - email: used for email notifications
    - referrer: overriding the referrer header value
    - next: url for redirecting
    - <name>~: for error messages
    """

    # Referrer
    data = defaultdict(list)
    for k, v in parse_qsl(urlparse(request.headers.get('REFERER')).query, keep_blank_values=True):
        data[k].append(v)

    # Request
    for arg in request.args:
        data[arg] = request.args.getlist(arg) or data[arg]

    # Body
    data.update(request.get_json() or {})

    # Form
    for field in request.form:
        data[field] = request.form.getlist(field) or data[field]

    # Flatten single item lists
    for k in data:
        if isinstance(data[k], list) and len(data[k]) == 1:
            data[k] = data[k][0]

    # remove `next`
    try:
        del data['next']
    except KeyError:
        pass

    ip_address = request.remote_addr if request.remote_addr != '127.0.0.1' \
        else request.headers.get('X-Forwarded-For')

    submission = Submission(
        data=dict(data),
        endpoint_id=endpoint.id,
        referrer=request.headers.get('REFERER'),
        user_agent=request.args.get('USER-AGENT'),
        ip_address=ip_address,
    )
    db.session.add(submission)
    db.session.commit()

    from formendpoint.tasks import process_submission
    process_submission.delay(submission.id)

    if 'next' in request.args:
        return redirect(request.args.get('next'))
    else:
        return redirect(url_for('endpoint', endpoint_id=endpoint.id, _external=True))


@login_manager.user_loader
def load_user(user_id):
    return User.query.get(user_id)


login_manager.login_view = "app.login"


@app.route('/')
def index():
    if current_user.is_authenticated:
        endpoint = Endpoint.query.filter_by(organization=current_user.organization).first()
        if endpoint:
            return redirect(url_for('endpoint', endpoint_id=endpoint.id))
        return redirect(url_for('create_endpoint'))

    else:
        url = furl(url_for('endpoint', endpoint_id=1, _external=True))
        url.args['destination'] = DEMO_URL
        form = render_template('form.html', url=url.url, input='<input type="email" name="email">')
        return render_template('welcome.html', form=form, demo_url=DEMO_URL)


@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'), 'favicon.ico',
{'fraction_non_alphanumeric': 0.10480920020909566, 'fraction_numerical': 0.00575013068478829, 'mean_word_length': 3.3606837606837607, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '771303', 'n_tokens_mistral': 2480, 'n_tokens_neox': 2425, 'n_words': 561}