<commit_before>import os
from datetime import datetime
import urllib2
from copy import deepcopy
import unicodedata
import httplib

from flask import Flask, jsonify, json, request, redirect, abort, make_response
from flask import render_template
from flask.views import View, MethodView
from flaskext.login import login_user, current_user

import bibserver.dao
from bibserver.config import config
import bibserver.iomanager
import bibserver.importer
from bibserver.core import app, login_manager
from bibserver.view.account import blueprint as account

app.register_blueprint(account, url_prefix='/account')


# NB: the decorator appears to kill the function for normal usage
@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = bibserver.dao.Account.get(userid)
    return out

@app.context_processor
def set_current_user():
    """ Set some template context globals. """
    return dict(current_user=current_user)


@app.route('/')
def home():
    # get list of available collections
    try:
        result = bibserver.dao.Record.query(q="type.exact:collection",sort={"received.exact":{"order":"desc"}})
        if result["hits"]["total"] != 0:
            colls = [i["_source"]["collection"] for i in result["hits"]["hits"]]
    except:
        colls = None
    return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )


@app.route('/content/<path:path>')
def content(path):
    return render_template('home/content.html', page=path)


@app.route('/collection/<collid>/<path:path>')
def record(collid,path):
    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    res = bibserver.dao.Record.query(q='collection:"' + collid + '" AND citekey:"' + path + '"')
    if res["hits"]["total"] == 0:
        abort(404)
    if res["hits"]["total"] > 1:
        return render_template('record.html', msg="hmmm... there is more than one record in this collection with that id...")
    recorddict = res["hits"]["hits"][0]["_source"]
    
    if JSON:
        resp = make_response( json.dumps( recorddict, indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('record.html', record=recorddict)


@app.route('/query', methods=['GET','POST'])
def query():
    if request.method == "GET":
        resp = make_response( bibserver.dao.Record.raw_query(request.query_string) )
        if request.values.get('delete','') and request.values.get('q',''):
            resp = make_response( bibserver.dao.Record.delete_by_query(request.values.get('q')) )
        resp.mimetype = "application/json"
        return resp

    if request.method == "POST":
        if request.values.get('callback',''):
            # this will fail if a POST is done to a URL that has more than just a callback param in the GET
            # why is it so hard to get the POST data object out of flask? Is there not a key for it? 
            callback = request.values["callback"]
            data = json.dumps(dict(request.form).keys()[1])
        else:
            data = json.dumps(dict(request.form).keys()[0])
            callback = ""
        host = str(config['ELASTIC_SEARCH_HOST']).rstrip('/')
        db_name = config['ELASTIC_SEARCH_DB']
        fullpath = '/' + db_name + '/record/_search?'
        if callback:
            path += 'callback=' + callback
        c =  httplib.HTTPConnection(host)
        c.request('POST', fullpath, data)
        resp = make_response(c.getresponse().read())
        resp.mimetype = "application/json"
        return resp

class UploadView(MethodView):
    '''The upload view.

    upload from URL provided in source, or from file upload button, or from
    POST default format is bibtex, but accept other format specifications via
    format default upload is a collection, but could also be person or group
    record
    '''
    def get(self):
        if request.values.get("source") is not None:
            return self.post()
        return render_template('upload.html')

    def post(self):
        pkg = self.package()
        if self.validate(pkg):
            importer = bibserver.importer.Importer()
#            try:
            res = importer.upload(pkg)
            if res != "DUPLICATE":<commit_msg>more testing<commit_after>import os
from datetime import datetime
import urllib2
from copy import deepcopy
import unicodedata
import httplib

from flask import Flask, jsonify, json, request, redirect, abort, make_response
from flask import render_template
from flask.views import View, MethodView
from flaskext.login import login_user, current_user

import bibserver.dao
from bibserver.config import config
import bibserver.iomanager
import bibserver.importer
from bibserver.core import app, login_manager
from bibserver.view.account import blueprint as account

app.register_blueprint(account, url_prefix='/account')


# NB: the decorator appears to kill the function for normal usage
@login_manager.user_loader
def load_account_for_login_manager(userid):
    out = bibserver.dao.Account.get(userid)
    return out

@app.context_processor
def set_current_user():
    """ Set some template context globals. """
    return dict(current_user=current_user)


@app.route('/')
def home():
    # get list of available collections
    try:
        result = bibserver.dao.Record.query(q="type.exact:collection",sort={"received.exact":{"order":"desc"}})
        if result["hits"]["total"] != 0:
            colls = [i["_source"]["collection"] for i in result["hits"]["hits"]]
    except:
        colls = None
    return render_template('home/index.html', colls=colls, upload=config["allow_upload"] )


@app.route('/content/<path:path>')
def content(path):
    return render_template('home/content.html', page=path)


@app.route('/collection/<collid>/<path:path>')
def record(collid,path):
    JSON = False
    if path.endswith(".json") or path.endswith(".bibjson") or request.values.get('format',"") == "json" or request.values.get('format',"") == "bibjson":
        path = path.replace(".bibjson","").replace(".json","")
        JSON = True

    res = bibserver.dao.Record.query(q='collection:"' + collid + '" AND citekey:"' + path + '"')
    if res["hits"]["total"] == 0:
        abort(404)
    if res["hits"]["total"] > 1:
        return render_template('record.html', msg="hmmm... there is more than one record in this collection with that id...")
    recorddict = res["hits"]["hits"][0]["_source"]
    
    if JSON:
        resp = make_response( json.dumps( recorddict, indent=4 ) )
        resp.mimetype = "application/json"
        return resp

    return render_template('record.html', record=recorddict)


@app.route('/query', methods=['GET','POST'])
def query():
    if request.method == "GET":
        resp = make_response( bibserver.dao.Record.raw_query(request.query_string) )
        if request.values.get('delete','') and request.values.get('q',''):
            resp = make_response( bibserver.dao.Record.delete_by_query(request.values.get('q')) )
        resp.mimetype = "application/json"
        return resp

    if request.method == "POST":
        if request.values.get('callback',''):
            # this will fail if a POST is done to a URL that has more than just a callback param in the GET
            # why is it so hard to get the POST data object out of flask? Is there not a key for it? 
            callback = request.values["callback"]
            data = json.dumps(dict(request.form).keys()[1])
        else:
            data = json.dumps(dict(request.form).keys()[0])
            callback = ""
        host = str(config['ELASTIC_SEARCH_HOST']).rstrip('/')
        db_name = config['ELASTIC_SEARCH_DB']
        fullpath = '/' + db_name + '/record/_search?'
        if callback:
            fullpath += 'callback=' + callback
        c =  httplib.HTTPConnection(host)
        c.request('POST', fullpath, data)
        resp = make_response(c.getresponse().read())
        resp.mimetype = "application/json"
        return resp

class UploadView(MethodView):
    '''The upload view.

    upload from URL provided in source, or from file upload button, or from
    POST default format is bibtex, but accept other format specifications via
    format default upload is a collection, but could also be person or group
    record
    '''
    def get(self):
        if request.values.get("source") is not None:
            return self.post()
        return render_template('upload.html')

    def post(self):
        pkg = self.package()
        if self.validate(pkg):
            importer = bibserver.importer.Importer()
#            try:
            res = importer.upload(pkg)
            if res != "DUPLICATE":
{'fraction_non_alphanumeric': 0.1319624901141114, 'fraction_numerical': 0.0024855948480397693, 'mean_word_length': 3.1794145420207744, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2682732', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2598, 'n_words': 710}