<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by <NAME>, <NAME>, <NAME> and <NAME>
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from flask import render_template, flash, request, Markup, \
    session, redirect, url_for, escape, Response, abort, send_file
import StringIO
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import (LoginManager, current_user, login_required,
                            login_user, logout_user, UserMixin, AnonymousUser,
                            confirm_login, fresh_login_required)
from sqlalchemy import func
from datetime import datetime

from iatidataquality import app
from iatidataquality import db

import os
import sys
import json

current = os.path.dirname(os.path.abspath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from iatidq import dqdownload, dqregistry, dqindicators, dqorganisations, dqpackages
from iatidq.models import *
import aggregation

import StringIO
import unicodecsv
import tempfile
import spreadsheet

def get_pc(pc_id):
    return db.session.query(
        PublisherCondition.id,
        PublisherCondition.description,
        PublisherCondition.operation,
        PublisherCondition.condition,
        PublisherCondition.condition_value,
        PackageGroup.title.label("packagegroup_name"),
        PackageGroup.id.label("packagegroup_id"),
        Test.name.label("test_name"),    
        Test.description.label("test_description"),
        Test.id.label("test_id")
        ).filter_by(id=pc_id
                    ).join(PackageGroup, Test).first()

def get_pcs():
    return db.session.query(
        PublisherCondition.id,
        PublisherCondition.description,
        PackageGroup.title.label("packagegroup_name"),
        PackageGroup.id.label("packagegroup_id"),
        Test.name.label("test_name"),    
        Test.description.label("test_description"),
        Test.id.label("test_id")
        ).order_by(
        PublisherCondition.id
        ).join(PackageGroup, Test
               ).all()
        
@app.route("/publisher_conditions/")
@app.route("/publisher_conditions/<id>/")
def publisher_conditions(id=None):
    if id is not None:
        pc = get_pc(id)
        return render_template("publisher_condition.html", pc=pc)
    else:
        pcs = get_pcs()
        return render_template("publisher_conditions.html", pcs=pcs)

def configure_publisher_condition(pc):
    pc.description = request.form['description']
    pc.publisher_id = int(request.form['publisher_id'])
    pc.test_id = int(request.form['test_id'])
    pc.operation = int(request.form['operation'])
    pc.condition = request.form['condition']
    pc.condition_value = request.form['condition_value']
    pc.file = request.form['file']
    pc.line = int(request.form['line'])
    pc.active = int(request.form['active'])

def update_publisher_condition(pc_id):
    pc = PublisherCondition.query.filter_by(id=pc_id).first_or_404()
    configure_publisher_condition(pc)
    db.session.add(pc)
    db.session.commit()

@app.route("/publisher_conditions/<id>/edit/", methods=['GET', 'POST'])
def publisher_conditions_editor(id=None):
    publishers = PackageGroup.query.order_by(
        PackageGroup.id).all()
    tests = Test.query.order_by(Test.id).all()
    if request.method == 'POST':
        if request.form['password'] == app.config["SECRET_PASSWORD"]:
            update_publisher_condition(id)
            flash('Updated', "success")
            return redirect(url_for('publisher_conditions_editor', id=pc.id))
        else:
            flash('Incorrect password', "error")
    else:
        pc = PublisherCondition.query.filter_by(id=id).first_or_404()
        return render_template("publisher_condition_editor.html", 
                               pc=pc, publishers=publishers, tests=tests)

@app.route("/publisher_conditions/new/", methods=['GET', 'POST'])
def publisher_conditions_new(id=None):
    publishers = PackageGroup.query.order_by(
        PackageGroup.id).all()
    tests = Test.query.order_by(Test.id).all()

    template_args = dict(
        pc={},
        publishers=publishers, 
        tests=tests
        )

    if (request.method == 'POST'):
        pc = PublisherCondition()
        configure_publisher_condition(pc)
        if (request.form['password'] == app.config["SECRET_PASSWORD"]):
            db.session.add(pc)
            db.session.commit()
            flash('Created new condition', "success")
            return redirect(url_for('publisher_conditions_editor', id=pc.id))
        else:
            flash('Incorrect password', "error")
            template_args["pc"] = pc
    else:
        return render_template("publisher_condition_editor.html", 
                               **template_args)

def ipc_step2():
    step = 2
    if (request.method == 'POST'):
        from iatidq import dqimportpublisherconditions
        if (request.form['password'] == app.config["SECRET_PASSWORD"]):
            if (request.form.get('local')):
                results = dqimportpublisherconditions.importPCsFromFile()<commit_msg>get type right<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by <NAME>, <NAME>, <NAME> and <NAME>
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from flask import render_template, flash, request, Markup, \
    session, redirect, url_for, escape, Response, abort, send_file
import StringIO
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import (LoginManager, current_user, login_required,
                            login_user, logout_user, UserMixin, AnonymousUser,
                            confirm_login, fresh_login_required)
from sqlalchemy import func
from datetime import datetime

from iatidataquality import app
from iatidataquality import db

import os
import sys
import json

current = os.path.dirname(os.path.abspath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from iatidq import dqdownload, dqregistry, dqindicators, dqorganisations, dqpackages
from iatidq.models import *
import aggregation

import StringIO
import unicodecsv
import tempfile
import spreadsheet

def get_pc(pc_id):
    return db.session.query(
        PublisherCondition.id,
        PublisherCondition.description,
        PublisherCondition.operation,
        PublisherCondition.condition,
        PublisherCondition.condition_value,
        PackageGroup.title.label("packagegroup_name"),
        PackageGroup.id.label("packagegroup_id"),
        Test.name.label("test_name"),    
        Test.description.label("test_description"),
        Test.id.label("test_id")
        ).filter_by(id=pc_id
                    ).join(PackageGroup, Test).first()

def get_pcs():
    return db.session.query(
        PublisherCondition.id,
        PublisherCondition.description,
        PackageGroup.title.label("packagegroup_name"),
        PackageGroup.id.label("packagegroup_id"),
        Test.name.label("test_name"),    
        Test.description.label("test_description"),
        Test.id.label("test_id")
        ).order_by(
        PublisherCondition.id
        ).join(PackageGroup, Test
               ).all()
        
@app.route("/publisher_conditions/")
@app.route("/publisher_conditions/<id>/")
def publisher_conditions(id=None):
    if id is not None:
        pc = get_pc(id)
        return render_template("publisher_condition.html", pc=pc)
    else:
        pcs = get_pcs()
        return render_template("publisher_conditions.html", pcs=pcs)

def configure_publisher_condition(pc):
    pc.description = request.form['description']
    pc.publisher_id = int(request.form['publisher_id'])
    pc.test_id = int(request.form['test_id'])
    pc.operation = int(request.form['operation'])
    pc.condition = request.form['condition']
    pc.condition_value = request.form['condition_value']
    pc.file = request.form['file']
    pc.line = int(request.form['line'])
    pc.active = int(request.form['active'])

def update_publisher_condition(pc_id):
    pc = PublisherCondition.query.filter_by(id=pc_id).first_or_404()
    configure_publisher_condition(pc)
    db.session.add(pc)
    db.session.commit()

@app.route("/publisher_conditions/<id>/edit/", methods=['GET', 'POST'])
def publisher_conditions_editor(id=None):
    publishers = PackageGroup.query.order_by(
        PackageGroup.id).all()
    tests = Test.query.order_by(Test.id).all()
    if request.method == 'POST':
        if request.form['password'] == app.config["SECRET_PASSWORD"]:
            update_publisher_condition(id)
            flash('Updated', "success")
            return redirect(url_for('publisher_conditions_editor', id=pc.id))
        else:
            flash('Incorrect password', "error")
    else:
        pc = PublisherCondition.query.filter_by(id=id).first_or_404()
        return render_template("publisher_condition_editor.html", 
                               pc=pc, publishers=publishers, tests=tests)

@app.route("/publisher_conditions/new/", methods=['GET', 'POST'])
def publisher_conditions_new(id=None):
    publishers = PackageGroup.query.order_by(
        PackageGroup.id).all()
    tests = Test.query.order_by(Test.id).all()

    template_args = dict(
        pc={},
        publishers=publishers, 
        tests=tests
        )

    if (request.method == 'POST'):
        pc = PublisherCondition()
        configure_publisher_condition(pc)
        if (request.form['password'] == app.config["SECRET_PASSWORD"]):
            db.session.add(pc)
            db.session.commit()
            flash('Created new condition', "success")
            return redirect(url_for('publisher_conditions_editor', id=pc.id))
        else:
            flash('Incorrect password', "error")
            template_args["pc"] = pc
    else:
        return render_template("publisher_condition_editor.html", 
                               **template_args)

def ipc_step2():
    step = '2'
    if (request.method == 'POST'):
        from iatidq import dqimportpublisherconditions
        if (request.form['password'] == app.config["SECRET_PASSWORD"]):
            if (request.form.get('local')):
                results = dqimportpublisherconditions.importPCsFromFile()
{'fraction_non_alphanumeric': 0.11643639828489757, 'fraction_numerical': 0.00266793711291091, 'mean_word_length': 3.1209265802905377, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3298415', 'n_tokens_mistral': 3114, 'n_tokens_neox': 3044, 'n_words': 603}