<commit_before>from functools import partial
from logging import StreamHandler
from os.path import isdir, isfile, join, basename
from sys import stderr
from time import time
from urllib import quote_plus

from flask import (Blueprint, Flask, send_from_directory, current_app,
                   send_file, request, redirect, jsonify, render_template,
                   url_for)
from pyelasticsearch import ElasticSearch
from werkzeug.exceptions import NotFound

from dxr.build import linked_pathname
from dxr.exceptions import BadTerm
from dxr.mime import icon
from dxr.plugins import FILE
from dxr.query import Query, filter_menu_items
from dxr.utils import non_negative_int, search_url, TEMPLATE_DIR, decode_es_datetime  # Make sure we load trilite before possibly importing the wrong version of sqlite3.


# Look in the 'dxr' package for static files, etc.:
dxr_blueprint = Blueprint('dxr_blueprint',
                          'dxr',
                          template_folder=TEMPLATE_DIR,
                          # static_folder seems to register a "static" route
                          # with the blueprint so the url_prefix (set later)
                          # takes effect for static files when found through
                          # url_for('static', ...).
                          static_folder='static')


def make_app(instance_path):
    """Return a DXR application which looks in the given folder for
    configuration.

    Also set up the static and template folder.

    """
    app = Flask('dxr', instance_path=instance_path)

    # Load the special config file generated by dxr-build:
    app.config.from_pyfile(join(app.instance_path, 'config.py'))

    app.register_blueprint(dxr_blueprint, url_prefix=app.config['WWW_ROOT'])

    # Log to Apache's error log in production:
    app.logger.addHandler(StreamHandler(stderr))

    # Make an ES connection pool shared among all threads:
    app.es = ElasticSearch(app.config['ES_HOSTS'])

    return app


@dxr_blueprint.route('/')
def index():
    config = current_app.config
    return redirect(url_for('.browse', tree=config['DEFAULT_TREE']))


@dxr_blueprint.route('/<tree>/search')
def search(tree):
    """Normalize params, and dispatch between JSON- and HTML-returning
    searches, based on Accept header.

    """
    # Normalize querystring params:
    config = current_app.config
    if tree not in config['TREES']:
        raise NotFound('No such tree as %s' % tree)
    req = request.values
    query_text = req.get('q', '')
    offset = non_negative_int(req.get('offset'), 0)
    limit = min(non_negative_int(req.get('limit'), 100), 1000)
    is_case_sensitive = req.get('case') == 'true'

    # Make a Query:
    query = Query(partial(current_app.es.search,
                          index=config['ES_ALIASES'][tree]),
                  query_text,
                  is_case_sensitive=is_case_sensitive)

    # Fire off one of the two search routines:
    searcher = _search_json if _request_wants_json() else _search_html
    return searcher(query, tree, query_text, is_case_sensitive, offset, limit, config)


def _search_json(query, tree, query_text, is_case_sensitive, offset, limit, config):
    """Do a normal search, and return the results as JSON."""
    try:
        # Convert to dicts for ease of manipulation in JS:
        results = [{'icon': icon,
                    'path': path,
                    'lines': [{'line_number': nb, 'line': l} for nb, l in lines]}
                   for icon, path, lines in query.results(offset, limit)]
    except BadTerm as exc:
        return jsonify({'error_html': exc.reason, 'error_level': 'warning'}), 400

    return jsonify({
        'wwwroot': config['WWW_ROOT'],
        'tree': tree,
        'results': results,
        'tree_tuples': _tree_tuples(config['TREES'], tree, query_text, is_case_sensitive)})


def _search_html(query, tree, query_text, is_case_sensitive, offset, limit, config):
    """Search a few different ways, and return the results as HTML.

    Try a "direct search" (for exact identifier matches, etc.). If that
    doesn't work, fall back to a normal search.

    """
    should_redirect = request.values.get('redirect') == 'true'

    # Try for a direct result:
    if should_redirect:  # always true in practice?
        result = query.direct_result()
        if result:
            path, line = result
            # TODO: Does this escape query_text properly?
            return redirect(
                '%s/%s/source/%s?from=%s%s#%i' %
                (config['WWW_ROOT'],
                 tree,
                 path,
                 query_text,
                 '&case=true' if is_case_sensitive else '',
                 line))

    # Try a normal search:
    template_vars = {
            'filters': filter_menu_items(),
            'generated_date': config['GENERATED_DATE'],
            'is_case_sensitive': is_case_sensitive,
            'query': query_text,
            'search_url': url_for('.search',
                                  tree=tree,
                                  q=query_text,
                                  redirect='false'),
            'tree': tree,
            'tree_tuples': _tree_tuples(config['TREES'], tree, query_text, is_case_sensitive),
            'wwwroot': config['WWW_ROOT']}

    try:
        results = list(query.results(offset, limit))
    except BadTerm as exc:
        return render_template('error.html',
                               results=[],
                               error_html=exc.reason,
                               **template_vars), 400

    return render_template('search.html', results=results, **template_vars)


def _tree_tuples(trees, tree, query_text, is_case_sensitive):
    return [(t,
             url_for('.search',
                     tree=t,
                     q=query_text,
                     **({'case': 'true'} if is_case_sensitive else {})),
             description)<commit_msg>Remove unneeded result= kwarg from error template call.<commit_after>from functools import partial
from logging import StreamHandler
from os.path import isdir, isfile, join, basename
from sys import stderr
from time import time
from urllib import quote_plus

from flask import (Blueprint, Flask, send_from_directory, current_app,
                   send_file, request, redirect, jsonify, render_template,
                   url_for)
from pyelasticsearch import ElasticSearch
from werkzeug.exceptions import NotFound

from dxr.build import linked_pathname
from dxr.exceptions import BadTerm
from dxr.mime import icon
from dxr.plugins import FILE
from dxr.query import Query, filter_menu_items
from dxr.utils import non_negative_int, search_url, TEMPLATE_DIR, decode_es_datetime  # Make sure we load trilite before possibly importing the wrong version of sqlite3.


# Look in the 'dxr' package for static files, etc.:
dxr_blueprint = Blueprint('dxr_blueprint',
                          'dxr',
                          template_folder=TEMPLATE_DIR,
                          # static_folder seems to register a "static" route
                          # with the blueprint so the url_prefix (set later)
                          # takes effect for static files when found through
                          # url_for('static', ...).
                          static_folder='static')


def make_app(instance_path):
    """Return a DXR application which looks in the given folder for
    configuration.

    Also set up the static and template folder.

    """
    app = Flask('dxr', instance_path=instance_path)

    # Load the special config file generated by dxr-build:
    app.config.from_pyfile(join(app.instance_path, 'config.py'))

    app.register_blueprint(dxr_blueprint, url_prefix=app.config['WWW_ROOT'])

    # Log to Apache's error log in production:
    app.logger.addHandler(StreamHandler(stderr))

    # Make an ES connection pool shared among all threads:
    app.es = ElasticSearch(app.config['ES_HOSTS'])

    return app


@dxr_blueprint.route('/')
def index():
    config = current_app.config
    return redirect(url_for('.browse', tree=config['DEFAULT_TREE']))


@dxr_blueprint.route('/<tree>/search')
def search(tree):
    """Normalize params, and dispatch between JSON- and HTML-returning
    searches, based on Accept header.

    """
    # Normalize querystring params:
    config = current_app.config
    if tree not in config['TREES']:
        raise NotFound('No such tree as %s' % tree)
    req = request.values
    query_text = req.get('q', '')
    offset = non_negative_int(req.get('offset'), 0)
    limit = min(non_negative_int(req.get('limit'), 100), 1000)
    is_case_sensitive = req.get('case') == 'true'

    # Make a Query:
    query = Query(partial(current_app.es.search,
                          index=config['ES_ALIASES'][tree]),
                  query_text,
                  is_case_sensitive=is_case_sensitive)

    # Fire off one of the two search routines:
    searcher = _search_json if _request_wants_json() else _search_html
    return searcher(query, tree, query_text, is_case_sensitive, offset, limit, config)


def _search_json(query, tree, query_text, is_case_sensitive, offset, limit, config):
    """Do a normal search, and return the results as JSON."""
    try:
        # Convert to dicts for ease of manipulation in JS:
        results = [{'icon': icon,
                    'path': path,
                    'lines': [{'line_number': nb, 'line': l} for nb, l in lines]}
                   for icon, path, lines in query.results(offset, limit)]
    except BadTerm as exc:
        return jsonify({'error_html': exc.reason, 'error_level': 'warning'}), 400

    return jsonify({
        'wwwroot': config['WWW_ROOT'],
        'tree': tree,
        'results': results,
        'tree_tuples': _tree_tuples(config['TREES'], tree, query_text, is_case_sensitive)})


def _search_html(query, tree, query_text, is_case_sensitive, offset, limit, config):
    """Search a few different ways, and return the results as HTML.

    Try a "direct search" (for exact identifier matches, etc.). If that
    doesn't work, fall back to a normal search.

    """
    should_redirect = request.values.get('redirect') == 'true'

    # Try for a direct result:
    if should_redirect:  # always true in practice?
        result = query.direct_result()
        if result:
            path, line = result
            # TODO: Does this escape query_text properly?
            return redirect(
                '%s/%s/source/%s?from=%s%s#%i' %
                (config['WWW_ROOT'],
                 tree,
                 path,
                 query_text,
                 '&case=true' if is_case_sensitive else '',
                 line))

    # Try a normal search:
    template_vars = {
            'filters': filter_menu_items(),
            'generated_date': config['GENERATED_DATE'],
            'is_case_sensitive': is_case_sensitive,
            'query': query_text,
            'search_url': url_for('.search',
                                  tree=tree,
                                  q=query_text,
                                  redirect='false'),
            'tree': tree,
            'tree_tuples': _tree_tuples(config['TREES'], tree, query_text, is_case_sensitive),
            'wwwroot': config['WWW_ROOT']}

    try:
        results = list(query.results(offset, limit))
    except BadTerm as exc:
        return render_template('error.html',
                               error_html=exc.reason,
                               **template_vars), 400

    return render_template('search.html', results=results, **template_vars)


def _tree_tuples(trees, tree, query_text, is_case_sensitive):
    return [(t,
             url_for('.search',
                     tree=t,
                     q=query_text,
                     **({'case': 'true'} if is_case_sensitive else {})),
             description)
{'fraction_non_alphanumeric': 0.09987389659520807, 'fraction_numerical': 0.0025220680958385876, 'mean_word_length': 2.348156487475373, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5338846', 'n_tokens_mistral': 3414, 'n_tokens_neox': 3271, 'n_words': 967}