<commit_before># encoding: utf-8
import os.path
import flask
import blinker
import schema
import widgets
from storage import get_db, Or, And, AllowWildcards, StorageError, NotFound
import statistics


webpages = flask.Blueprint('webpages', __name__)


@webpages.route('/')
def index():
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    return flask.render_template('index.html', form=form,
                                 search_form=schema.Search())


def _other_site_labels(doc):
    db = get_db()
    other_site_ids = Or(o.value for o in doc['section1']['other_sites'])
    results = db.search({'id': other_site_ids})
    return dict((data['id'], data['name']) for data in results['docs'])


@webpages.route('/view')
def view():
    doc_id = flask.request.args.get('doc_id')
    db = get_db()
    try:
        doc = db.load_document(doc_id)
    except NotFound:
        flask.abort(404)
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    form.other_site_labels = _other_site_labels(doc)
    options = {
        'title': _doc_title(doc),
        'form': form,
        'doc_id': doc_id,
        'doc': doc,
    }

    app = flask.current_app
    if 'PDF_FOLDER' in app.config:
        pdf_name = doc_id.lower() + '.pdf'
        if os.path.isfile(os.path.join(app.config['PDF_FOLDER'], pdf_name)):
            options['pdf_url'] = flask.url_for('static',
                                               filename='pdf/' + pdf_name)
    return flask.render_template('view.html', **options)


def _doc_title(doc):
    return u"%s (%s)" % (doc['section1']['name'].u, doc['section1']['code'].u)


@webpages.route('/new', methods=['GET', 'POST'])
@webpages.route('/edit', methods=['GET', 'POST'])
def edit():
    doc_id = flask.request.args.get('doc_id', None)
    db = get_db()
    new_doc = bool(doc_id is None)

    if flask.request.method == 'POST':
        request_form = flask.request.form.to_dict()
        request_form["section1_code"] = doc_id
        doc = schema.SpaDoc.from_flat(request_form)

        if doc.validate():
            doc_id = db.save_document(doc_id, doc)
            app = flask.current_app
            app.document_signal.send('save', doc_id=doc_id, doc=doc)
            flask.flash("Document %r saved" % doc_id)
            return flask.redirect(flask.url_for('webpages.view', doc_id=doc_id))

        else:
            flask.flash("Errors in document")

    else:
        if new_doc:
            doc = schema.SpaDoc()
        else:
            doc = db.load_document(doc_id)

    title = u"Introducere sit nou" if new_doc else _doc_title(doc)
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    return flask.render_template('edit.html', title=title,
                                 doc=doc, form=form)


class SearchError(Exception):
    """ Error occurred during search """


def _db_search(search_form, **kwargs):
    db = get_db()
    query = search_form.value

    text_str = (query.pop('text') or u"").strip()
    if any(text_str.startswith(ch) for ch in ['*', '?']):
        # the Solr QueryParser does not accept leading-wildcard queries
        raise SearchError(u"Textul nu poate să înceapă cu '*' sau '?'")

    text = AllowWildcards(text_str)
    text_or = Or([('name', text), ('text', text)])
    full_query = And([text_or, query])

    try:
        return db.search(full_query, **kwargs)
    except StorageError, e:
        raise SearchError(u"Eroare bază de date: %s" % e)


@webpages.route('/search')
def search():
    form = widgets.SearchMarkupGenerator(flask.current_app.jinja_env)
    search_form = schema.Search.from_flat(flask.request.args.to_dict())

    try:
        search_answer = _db_search(search_form, facets=True)
    except SearchError, e:
        return flask.render_template('search_error.html', msg=unicode(e),
                                     form=form, search_form=search_form)

    search_answer['facets']['nuts3'] = [
        e for e in search_answer['facets']['nuts3']
          if e.name.startswith(search_form['nuts2'].value)]
    form['facets'] = search_answer['facets']

    return flask.render_template('search.html', form=form,
                                 search_form=search_form,
                                 search_answer=search_answer,
                                 stat_labels=statistics.label)


@webpages.route('/stats')
def stats():
    args = flask.request.args.to_dict()

    search_form = schema.Search.from_flat(args)
    stat_form = schema.Statistics.from_flat(args)
    get_data = statistics.need_data(stat_form)
<commit_msg>fix bug search_form['nuts2']<commit_after># encoding: utf-8
import os.path
import flask
import blinker
import schema
import widgets
from storage import get_db, Or, And, AllowWildcards, StorageError, NotFound
import statistics


webpages = flask.Blueprint('webpages', __name__)


@webpages.route('/')
def index():
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    return flask.render_template('index.html', form=form,
                                 search_form=schema.Search())


def _other_site_labels(doc):
    db = get_db()
    other_site_ids = Or(o.value for o in doc['section1']['other_sites'])
    results = db.search({'id': other_site_ids})
    return dict((data['id'], data['name']) for data in results['docs'])


@webpages.route('/view')
def view():
    doc_id = flask.request.args.get('doc_id')
    db = get_db()
    try:
        doc = db.load_document(doc_id)
    except NotFound:
        flask.abort(404)
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    form.other_site_labels = _other_site_labels(doc)
    options = {
        'title': _doc_title(doc),
        'form': form,
        'doc_id': doc_id,
        'doc': doc,
    }

    app = flask.current_app
    if 'PDF_FOLDER' in app.config:
        pdf_name = doc_id.lower() + '.pdf'
        if os.path.isfile(os.path.join(app.config['PDF_FOLDER'], pdf_name)):
            options['pdf_url'] = flask.url_for('static',
                                               filename='pdf/' + pdf_name)
    return flask.render_template('view.html', **options)


def _doc_title(doc):
    return u"%s (%s)" % (doc['section1']['name'].u, doc['section1']['code'].u)


@webpages.route('/new', methods=['GET', 'POST'])
@webpages.route('/edit', methods=['GET', 'POST'])
def edit():
    doc_id = flask.request.args.get('doc_id', None)
    db = get_db()
    new_doc = bool(doc_id is None)

    if flask.request.method == 'POST':
        request_form = flask.request.form.to_dict()
        request_form["section1_code"] = doc_id
        doc = schema.SpaDoc.from_flat(request_form)

        if doc.validate():
            doc_id = db.save_document(doc_id, doc)
            app = flask.current_app
            app.document_signal.send('save', doc_id=doc_id, doc=doc)
            flask.flash("Document %r saved" % doc_id)
            return flask.redirect(flask.url_for('webpages.view', doc_id=doc_id))

        else:
            flask.flash("Errors in document")

    else:
        if new_doc:
            doc = schema.SpaDoc()
        else:
            doc = db.load_document(doc_id)

    title = u"Introducere sit nou" if new_doc else _doc_title(doc)
    form = widgets.MarkupGenerator(flask.current_app.jinja_env)
    return flask.render_template('edit.html', title=title,
                                 doc=doc, form=form)


class SearchError(Exception):
    """ Error occurred during search """


def _db_search(search_form, **kwargs):
    db = get_db()
    query = search_form.value

    text_str = (query.pop('text') or u"").strip()
    if any(text_str.startswith(ch) for ch in ['*', '?']):
        # the Solr QueryParser does not accept leading-wildcard queries
        raise SearchError(u"Textul nu poate să înceapă cu '*' sau '?'")

    text = AllowWildcards(text_str)
    text_or = Or([('name', text), ('text', text)])
    full_query = And([text_or, query])

    try:
        return db.search(full_query, **kwargs)
    except StorageError, e:
        raise SearchError(u"Eroare bază de date: %s" % e)


@webpages.route('/search')
def search():
    form = widgets.SearchMarkupGenerator(flask.current_app.jinja_env)
    search_form = schema.Search.from_flat(flask.request.args.to_dict())

    try:
        search_answer = _db_search(search_form, facets=True)
    except SearchError, e:
        return flask.render_template('search_error.html', msg=unicode(e),
                                     form=form, search_form=search_form)

    if "nuts2" in search_form and search_form["nuts2"]:
        search_answer['facets']['nuts3'] = [
            e for e in search_answer['facets']['nuts3']
              if e.name.startswith(search_form['nuts2'].value)]
    form['facets'] = search_answer['facets']

    return flask.render_template('search.html', form=form,
                                 search_form=search_form,
                                 search_answer=search_answer,
                                 stat_labels=statistics.label)


@webpages.route('/stats')
def stats():
    args = flask.request.args.to_dict()

    search_form = schema.Search.from_flat(args)
    stat_form = schema.Statistics.from_flat(args)
    get_data = statistics.need_data(stat_form)

{'fraction_non_alphanumeric': 0.13567948023822415, 'fraction_numerical': 0.0027070925825663237, 'mean_word_length': 2.774417654270535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4062351', 'n_tokens_mistral': 3054, 'n_tokens_neox': 2940, 'n_words': 580}