<commit_before># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

import time
import re
import json
import os
import math
import requests
from StringIO import StringIO

from flask import Blueprint, request, url_for, flash, redirect, abort, Response, current_app
from flask import render_template, make_response, session
from flask.ext.login import login_required, current_user
from flask.ext.babel import gettext
from rq import Queue

import pybossa.sched as sched

from pybossa.core import (uploader, signer, sentinel, json_exporter,
                          csv_exporter, importer, sentinel)
from pybossa.model import make_uuid
from pybossa.model.project import Project
from pybossa.model.category import Category
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.auditlog import Auditlog
from pybossa.model.webhook import Webhook
from pybossa.model.blogpost import Blogpost
from pybossa.util import (Pagination, admin_required, get_user_id_or_ip, rank,
                          handle_content_type, redirect_content_type)
from pybossa.auth import ensure_authorized_to
from pybossa.cache import projects as cached_projects
from pybossa.cache import users as cached_users
from pybossa.cache import categories as cached_cat
from pybossa.cache import project_stats as stats
from pybossa.cache.helpers import add_custom_contrib_button_to, has_no_presenter
from pybossa.ckan import Ckan
from pybossa.extensions import misaka
from pybossa.cookies import CookieHandler
from pybossa.password_manager import ProjectPasswdManager
from pybossa.jobs import import_tasks, IMPORT_TASKS_TIMEOUT, webhook
from pybossa.forms.projects_view_forms import *
from pybossa.importers import BulkImportException
from pybossa.pro_features import ProFeatureHandler

from pybossa.core import (project_repo, user_repo, task_repo, blog_repo,
                          result_repo, webhook_repo, auditlog_repo)
from pybossa.auditlogger import AuditLogger
from pybossa.contributions_guard import ContributionsGuard

blueprint = Blueprint('project', __name__)

MAX_NUM_SYNCHRONOUS_TASKS_IMPORT = 200
auditlogger = AuditLogger(auditlog_repo, caller='web')
importer_queue = Queue('medium',
                       connection=sentinel.master,
                       default_timeout=IMPORT_TASKS_TIMEOUT)
webhook_queue = Queue('high', connection=sentinel.master)

def project_title(project, page_name):
    if not project:  # pragma: no cover
        return "Project not found"
    if page_name is None:
        return "Project: %s" % (project.name)
    return "Project: %s &middot; %s" % (project.name, page_name)


def project_by_shortname(short_name):
    project = cached_projects.get_project(short_name)
    if project:
        # Get owner
        owner = user_repo.get(project.owner_id)
        # Populate CACHE with the data of the project
        return (project,
                owner,
                cached_projects.n_tasks(project.id),
                cached_projects.n_task_runs(project.id),
                cached_projects.overall_progress(project.id),
                cached_projects.last_activity(project.id),
                cached_projects.n_results(project.id))
    else:
        cached_projects.delete_project(short_name)
        return abort(404)


def pro_features(owner=None):
    feature_handler = ProFeatureHandler(current_app.config.get('PRO_FEATURES'))
    pro = {
        'auditlog_enabled': feature_handler.auditlog_enabled_for(current_user),
        'autoimporter_enabled': feature_handler.autoimporter_enabled_for(current_user),
        'webhooks_enabled': feature_handler.webhooks_enabled_for(current_user)
    }
    if owner:
        pro['better_stats_enabled'] = feature_handler.better_stats_enabled_for(
                                          current_user,
                                          owner)
    return pro


@blueprint.route('/', defaults={'page': 1})
@blueprint.route('/page/<int:page>/', defaults={'page': 1})
def redirect_old_featured(page):
    """DEPRECATED only to redirect old links"""
    return redirect(url_for('.index', page=page), 301)


@blueprint.route('/published/', defaults={'page': 1})
@blueprint.route('/published/<int:page>/', defaults={'page': 1})
def redirect_old_published(page):  # pragma: no cover
    """DEPRECATED only to redirect old links"""
    category = project_repo.get_category()
    return redirect(url_for('.project_cat_index', category=category.short_name, page=page), 301)


@blueprint.route('/draft/', defaults={'page': 1})
@blueprint.route('/draft/<int:page>/', defaults={'page': 1})
def redirect_old_draft(page):
    """DEPRECATED only to redirect old links"""
    return redirect(url_for('.draft', page=page), 301)


@blueprint.route('/category/featured/', defaults={'page': 1})
@blueprint.route('/category/featured/page/<int:page>/')
def index(page):
    """List projects in the system"""
    if cached_projects.n_count('featured') > 0:
        return project_index(page, cached_projects.get_all_featured,
                             'featured',
                             True, False)
    else:
        categories = cached_cat.get_all()
        cat_short_name = categories[0].short_name
        return redirect_content_type(url_for('.project_cat_index', category=cat_short_name))


def project_index(page, lookup, category, fallback, use_count):
    """Show projects of a category"""

    per_page = current_app.config['APPS_PER_PAGE']

    ranked_projects = rank(lookup(category))
    offset = (page - 1) * per_page
    projects = ranked_projects[offset:offset+per_page]

    count = cached_projects.n_count(category)

    data = []

    if fallback and not projects:  # pragma: no cover
        return redirect(url_for('.index'))

    pagination = Pagination(page, per_page, count)
    categories = cached_cat.get_all()
    # Check for pre-defined categories featured and draft
    featured_cat = Category(name='Featured',
                            short_name='featured',
                            description='Featured projects')
    if category == 'featured':
        active_cat = featured_cat
    elif category == 'draft':
        active_cat = Category(name='Draft',
                              short_name='draft',
                              description='Draft projects')
    else:
        active_cat = project_repo.get_category_by(short_name=category)

    # Check if we have to add the section Featured to local nav
    if cached_projects.n_count('featured') > 0:
        categories.insert(0, featured_cat)
    template_args = {
        "projects": projects,
        "title": gettext("Projects"),
        "pagination": pagination,
        "active_cat": active_cat,
        "categories": categories,
        "template": '/projects/index.html'}

    if use_count:
        template_args.update({"count": count})
    return handle_content_type(template_args)


@blueprint.route('/category/draft/', defaults={'page': 1})
@blueprint.route('/category/draft/page/<int:page>/')
@login_required
@admin_required
def draft(page):
    """Show the Draft projects"""
    return project_index(page, cached_projects.get_all_draft, 'draft',
                     False, True)


@blueprint.route('/category/<string:category>/', defaults={'page': 1})
@blueprint.route('/category/<string:category>/page/<int:page>/')
def project_cat_index(category, page):
    """Show Projects that belong to a given category"""
    return project_index(page, cached_projects.get_all, category, False, True)


@blueprint.route('/new', methods=['GET', 'POST'])
@login_required
def new():
    ensure_authorized_to('create', Project)
    form = ProjectForm(request.form)

    def respond(errors):
        return render_template('projects/new.html',
                               title=gettext("Create a Project"),
                               form=form, errors=errors)

    def _description_from_long_description():
        if form.description.data:
            return form.description.data
        long_desc = form.long_description.data
        html_long_desc = misaka.render(long_desc)[:-1]
        remove_html_tags_regex = re.compile('<[^>]*>')
        blank_space_regex = re.compile('\n')
        text_desc = remove_html_tags_regex.sub("", html_long_desc)[:255]
        if len(text_desc) >= 252:
            text_desc = text_desc[:-3]
            text_desc += "..."
        description = blank_space_regex.sub(" ", text_desc)
        return description if description else " "

    if request.method != 'POST':
        return respond(False)

    if not form.validate():
        flash(gettext('Please correct the errors'), 'error')
        return respond(True)

    info = {}
    category_by_default = cached_cat.get_all()[0]

    project = Project(name=form.name.data,
              short_name=form.short_name.data,
              description=_description_from_long_description(),
              long_description=form.long_description.data,
              owner_id=current_user.id,
              info=info,
              category_id=category_by_default.id)

    project_repo.save(project)

    msg_1 = gettext('Project created!')
    flash('<i class="icon-ok"></i> ' + msg_1, 'success')
    flash('<i class="icon-bullhorn"></i> ' +
          gettext('You can check the ') +
          '<strong><a href="https://docs.pybossa.com">' +
          gettext('Guide and Documentation') +
          '</a></strong> ' +
          gettext('for adding tasks, a thumbnail, using PYBOSSA.JS, etc.'),
          'info')
    auditlogger.add_log_entry(None, project, current_user)

    return redirect(url_for('.update', short_name=project.short_name))


@blueprint.route('/<short_name>/tasks/taskpresentereditor', methods=['GET', 'POST'])
@login_required
def task_presenter_editor(short_name):
    errors = False
    (project, owner, n_tasks, n_task_runs,
     overall_progress,
     last_activity, n_results) = project_by_shortname(short_name)

    title = project_title(project, "Task Presenter Editor")
    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)

    pro = pro_features()

    form = TaskPresenterForm(request.form)
    form.id.data = project.id
    if request.method == 'POST' and form.validate():
        db_project = project_repo.get(project.id)
        old_project = Project(**db_project.dictize())
        old_info = dict(db_project.info)
        old_info['task_presenter'] = form.editor.data
        db_project.info = old_info
        auditlogger.add_log_entry(old_project, db_project, current_user)
        project_repo.update(db_project)
        msg_1 = gettext('Task presenter added!')
        flash('<i class="icon-ok"></i> ' + msg_1, 'success')
        return redirect(url_for('.tasks', short_name=project.short_name))

    # It does not have a validation
    if request.method == 'POST' and not form.validate():  # pragma: no cover
        flash(gettext('Please correct the errors'), 'error')
        errors = True

    if project.info.get('task_presenter'):
        form.editor.data = project.info['task_presenter']
    else:
        if not request.args.get('template'):
            msg_1 = gettext('<strong>Note</strong> You will need to upload the'
                            ' tasks using the')
            msg_2 = gettext('CSV importer')
            msg_3 = gettext(' or download the project bundle and run the'
                            ' <strong>createTasks.py</strong> script in your'
                            ' computer')
            url = '<a href="%s"> %s</a>' % (url_for('project.import_task',
                                                    short_name=project.short_name), msg_2)
            msg = msg_1 + url + msg_3
            flash(msg, 'info')

            wrap = lambda i: "projects/presenters/%s.html" % i
            pres_tmpls = map(wrap, current_app.config.get('PRESENTERS'))

            project = add_custom_contrib_button_to(project, get_user_id_or_ip())
            return render_template(
                'projects/task_presenter_options.html',
                title=title,
                project=project,
                owner=owner,
                overall_progress=overall_progress,
                n_tasks=n_tasks,
                n_task_runs=n_task_runs,
                last_activity=last_activity,
                n_completed_tasks=cached_projects.n_completed_tasks(project.get('id')),
                n_volunteers=cached_projects.n_volunteers(project.get('id')),
                presenters=pres_tmpls,
                pro_features=pro)

        tmpl_uri = "projects/snippets/%s.html" \
            % request.args.get('template')
        tmpl = render_template(tmpl_uri, project=project)
        form.editor.data = tmpl
        msg = 'Your code will be <em>automagically</em> rendered in \
                      the <strong>preview section</strong>. Click in the \
                      preview button!'
        flash(gettext(msg), 'info')
    dict_project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    return render_template('projects/task_presenter_editor.html',
                           title=title,
                           form=form,
                           project=dict_project,
                           owner=owner,
                           overall_progress=overall_progress,
                           n_tasks=n_tasks,
                           n_task_runs=n_task_runs,
                           last_activity=last_activity,
                           n_completed_tasks=cached_projects.n_completed_tasks(project.id),
                           n_volunteers=cached_projects.n_volunteers(project.id),
                           errors=errors,
                           pro_features=pro)


@blueprint.route('/<short_name>/delete', methods=['GET', 'POST'])
@login_required
def delete(short_name):
    (project, owner, n_tasks,
    n_task_runs, overall_progress, last_activity,
    n_results) = project_by_shortname(short_name)
    title = project_title(project, "Delete")
    ensure_authorized_to('read', project)
    ensure_authorized_to('delete', project)
    pro = pro_features()
    if request.method == 'GET':
        return render_template('/projects/delete.html',
                               title=title,
                               project=project,
                               owner=owner,
                               n_tasks=n_tasks,
                               overall_progress=overall_progress,
                               last_activity=last_activity,
                               pro_features=pro)
    project_repo.delete(project)
    auditlogger.add_log_entry(project, None, current_user)
    flash(gettext('Project deleted!'), 'success')
    return redirect(url_for('account.profile', name=current_user.name))


@blueprint.route('/<short_name>/update', methods=['GET', 'POST'])
@login_required
def update(short_name):
    (project, owner, n_tasks,
     n_task_runs, overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    def handle_valid_form(form):
        (project, owner, n_tasks, n_task_runs,
         overall_progress, last_activity,
         n_results) = project_by_shortname(short_name)

        new_project = project_repo.get_by_shortname(short_name)
        old_project = Project(**new_project.dictize())
        old_info = dict(new_project.info)
        old_project.info = old_info
        if form.id.data == new_project.id:
            new_project.name = form.name.data
            new_project.short_name = form.short_name.data
            new_project.description = form.description.data
            new_project.long_description = form.long_description.data
            new_project.webhook = form.webhook.data
            new_project.info = project.info
            new_project.owner_id = project.owner_id
            new_project.allow_anonymous_contributors = form.allow_anonymous_contributors.data
            new_project.category_id = form.category_id.data

        if form.protect.data and form.password.data:
            new_project.set_password(form.password.data)
        if not form.protect.data:
            new_project.set_password("")

        project_repo.update(new_project)
        auditlogger.add_log_entry(old_project, new_project, current_user)
        cached_cat.reset()
        cached_projects.get_project(new_project.short_name)
        flash(gettext('Project updated!'), 'success')
        return redirect(url_for('.details',
                                short_name=new_project.short_name))

    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)

    pro = pro_features()

    title = project_title(project, "Update")
    if request.method == 'GET':
        form = ProjectUpdateForm(obj=project)
        upload_form = AvatarUploadForm()
        categories = project_repo.get_all_categories()
        form.category_id.choices = [(c.id, c.name) for c in categories]
        if project.category_id is None:
            project.category_id = categories[0].id
        form.populate_obj(project)
        form.protect.data = project.needs_password()

    if request.method == 'POST':
        upload_form = AvatarUploadForm()
        form = ProjectUpdateForm(request.form)
        categories = cached_cat.get_all()
        form.category_id.choices = [(c.id, c.name) for c in categories]

        if request.form.get('btn') != 'Upload':
            if form.validate():
                return handle_valid_form(form)
            flash(gettext('Please correct the errors'), 'error')
        else:
            if upload_form.validate_on_submit():
                project = project_repo.get(project.id)
                _file = request.files['avatar']
                coordinates = (upload_form.x1.data, upload_form.y1.data,
                               upload_form.x2.data, upload_form.y2.data)
                prefix = time.time()
                _file.filename = "project_%s_thumbnail_%i.png" % (project.id, prefix)
                container = "user_%s" % current_user.id
                uploader.upload_file(_file,
                                     container=container,
                                     coordinates=coordinates)
                # Delete previous avatar from storage
                if project.info.get('thumbnail'):
                    uploader.delete_file(project.info['thumbnail'], container)
                project.info['thumbnail'] = _file.filename
                project.info['container'] = container
                project_repo.save(project)
                flash(gettext('Your project thumbnail has been updated! It may \
                                  take some minutes to refresh...'), 'success')
            else:
                flash(gettext('You must provide a file to change the avatar'),
                      'error')
            return redirect(url_for('.update', short_name=short_name))

    project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    return render_template('/projects/update.html',
                           form=form,
                           upload_form=upload_form,
                           project=project,
                           owner=owner,
                           n_tasks=n_tasks,
                           overall_progress=overall_progress,
                           n_task_runs=n_task_runs,
                           last_activity=last_activity,
                           n_completed_tasks=cached_projects.n_completed_tasks(project.get('id')),
                           n_volunteers=cached_projects.n_volunteers(project.get('id')),
                           title=title,
                           pro_features=pro)


@blueprint.route('/<short_name>/')
def details(short_name):
    (project, owner, n_tasks, n_task_runs,
     overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    if project.needs_password():
        redirect_to_password = _check_if_redirect_to_password(project)
        if redirect_to_password:
            return redirect_to_password
    else:
        ensure_authorized_to('read', project)

    template = '/projects/project.html'
    pro = pro_features()

    title = project_title(project, None)
    project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    if current_user.is_authenticated() and owner.id == current_user.id:
        project_sanitized = project
        owner_sanitized = cached_users.get_user_summary(owner.name)
    else:   # anonymous or different owner
        if request.headers['Content-Type'] == 'application/json':
            project_sanitized = Project().to_public_json(project)
        else:    # HTML
            project_sanitized = project
        owner_sanitized = cached_users.public_get_user_summary(owner.name)
    template_args = {"project": project_sanitized,
                     "title": title,
                     "owner":  owner_sanitized,
                     "n_tasks": n_tasks,
                     "n_task_runs": n_task_runs,
                     "overall_progress": overall_progress,
                     "last_activity": last_activity,
                     "n_completed_tasks": cached_projects.n_completed_tasks(project.get('id')),
                     "n_volunteers": cached_projects.n_volunteers(project.get('id')),
                     "pro_features": pro}
    if current_app.config.get('CKAN_URL'):
        template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
        template_args['ckan_url'] = current_app.config.get('CKAN_URL')
        template_args['ckan_pkg_name'] = short_name
    response = dict(template=template, **template_args)
    return handle_content_type(response)


@blueprint.route('/<short_name>/settings')
@login_required
def settings(short_name):
    (project, owner, n_tasks, n_task_runs,
     overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    title = project_title(project, "Settings")
    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)
    pro = pro_features()
    project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    owner_serialized = cached_users.get_user_summary(owner.name)
    response = dict(template='/projects/settings.html',
                    project=project,
                    owner=owner_serialized,
                    n_tasks=n_tasks,
                    overall_progress=overall_progress,
                    n_task_runs=n_task_runs,
                    last_activity=last_activity,
                    n_completed_tasks=cached_projects.n_completed_tasks(
                        project.get('id')),
                    n_volunteers=cached_projects.n_volunteers(<commit_msg>sanitize output<commit_after># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

import time
import re
import json
import os
import math
import requests
from StringIO import StringIO

from flask import Blueprint, request, url_for, flash, redirect, abort, Response, current_app
from flask import render_template, make_response, session
from flask.ext.login import login_required, current_user
from flask.ext.babel import gettext
from rq import Queue

import pybossa.sched as sched

from pybossa.core import (uploader, signer, sentinel, json_exporter,
                          csv_exporter, importer, sentinel)
from pybossa.model import make_uuid
from pybossa.model.project import Project
from pybossa.model.category import Category
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.auditlog import Auditlog
from pybossa.model.webhook import Webhook
from pybossa.model.blogpost import Blogpost
from pybossa.util import (Pagination, admin_required, get_user_id_or_ip, rank,
                          handle_content_type, redirect_content_type)
from pybossa.auth import ensure_authorized_to
from pybossa.cache import projects as cached_projects
from pybossa.cache import users as cached_users
from pybossa.cache import categories as cached_cat
from pybossa.cache import project_stats as stats
from pybossa.cache.helpers import add_custom_contrib_button_to, has_no_presenter
from pybossa.ckan import Ckan
from pybossa.extensions import misaka
from pybossa.cookies import CookieHandler
from pybossa.password_manager import ProjectPasswdManager
from pybossa.jobs import import_tasks, IMPORT_TASKS_TIMEOUT, webhook
from pybossa.forms.projects_view_forms import *
from pybossa.importers import BulkImportException
from pybossa.pro_features import ProFeatureHandler

from pybossa.core import (project_repo, user_repo, task_repo, blog_repo,
                          result_repo, webhook_repo, auditlog_repo)
from pybossa.auditlogger import AuditLogger
from pybossa.contributions_guard import ContributionsGuard

blueprint = Blueprint('project', __name__)

MAX_NUM_SYNCHRONOUS_TASKS_IMPORT = 200
auditlogger = AuditLogger(auditlog_repo, caller='web')
importer_queue = Queue('medium',
                       connection=sentinel.master,
                       default_timeout=IMPORT_TASKS_TIMEOUT)
webhook_queue = Queue('high', connection=sentinel.master)

def project_title(project, page_name):
    if not project:  # pragma: no cover
        return "Project not found"
    if page_name is None:
        return "Project: %s" % (project.name)
    return "Project: %s &middot; %s" % (project.name, page_name)


def project_by_shortname(short_name):
    project = cached_projects.get_project(short_name)
    if project:
        # Get owner
        owner = user_repo.get(project.owner_id)
        # Populate CACHE with the data of the project
        return (project,
                owner,
                cached_projects.n_tasks(project.id),
                cached_projects.n_task_runs(project.id),
                cached_projects.overall_progress(project.id),
                cached_projects.last_activity(project.id),
                cached_projects.n_results(project.id))
    else:
        cached_projects.delete_project(short_name)
        return abort(404)


def pro_features(owner=None):
    feature_handler = ProFeatureHandler(current_app.config.get('PRO_FEATURES'))
    pro = {
        'auditlog_enabled': feature_handler.auditlog_enabled_for(current_user),
        'autoimporter_enabled': feature_handler.autoimporter_enabled_for(current_user),
        'webhooks_enabled': feature_handler.webhooks_enabled_for(current_user)
    }
    if owner:
        pro['better_stats_enabled'] = feature_handler.better_stats_enabled_for(
                                          current_user,
                                          owner)
    return pro


@blueprint.route('/', defaults={'page': 1})
@blueprint.route('/page/<int:page>/', defaults={'page': 1})
def redirect_old_featured(page):
    """DEPRECATED only to redirect old links"""
    return redirect(url_for('.index', page=page), 301)


@blueprint.route('/published/', defaults={'page': 1})
@blueprint.route('/published/<int:page>/', defaults={'page': 1})
def redirect_old_published(page):  # pragma: no cover
    """DEPRECATED only to redirect old links"""
    category = project_repo.get_category()
    return redirect(url_for('.project_cat_index', category=category.short_name, page=page), 301)


@blueprint.route('/draft/', defaults={'page': 1})
@blueprint.route('/draft/<int:page>/', defaults={'page': 1})
def redirect_old_draft(page):
    """DEPRECATED only to redirect old links"""
    return redirect(url_for('.draft', page=page), 301)


@blueprint.route('/category/featured/', defaults={'page': 1})
@blueprint.route('/category/featured/page/<int:page>/')
def index(page):
    """List projects in the system"""
    if cached_projects.n_count('featured') > 0:
        return project_index(page, cached_projects.get_all_featured,
                             'featured',
                             True, False)
    else:
        categories = cached_cat.get_all()
        cat_short_name = categories[0].short_name
        return redirect_content_type(url_for('.project_cat_index', category=cat_short_name))


def project_index(page, lookup, category, fallback, use_count):
    """Show projects of a category"""

    per_page = current_app.config['APPS_PER_PAGE']

    ranked_projects = rank(lookup(category))
    offset = (page - 1) * per_page
    projects = ranked_projects[offset:offset+per_page]

    count = cached_projects.n_count(category)

    data = []

    if fallback and not projects:  # pragma: no cover
        return redirect(url_for('.index'))

    pagination = Pagination(page, per_page, count)
    categories = cached_cat.get_all()
    # Check for pre-defined categories featured and draft
    featured_cat = Category(name='Featured',
                            short_name='featured',
                            description='Featured projects')
    if category == 'featured':
        active_cat = featured_cat
    elif category == 'draft':
        active_cat = Category(name='Draft',
                              short_name='draft',
                              description='Draft projects')
    else:
        active_cat = project_repo.get_category_by(short_name=category)

    # Check if we have to add the section Featured to local nav
    if cached_projects.n_count('featured') > 0:
        categories.insert(0, featured_cat)
    template_args = {
        "projects": projects,
        "title": gettext("Projects"),
        "pagination": pagination,
        "active_cat": active_cat,
        "categories": categories,
        "template": '/projects/index.html'}

    if use_count:
        template_args.update({"count": count})
    return handle_content_type(template_args)


@blueprint.route('/category/draft/', defaults={'page': 1})
@blueprint.route('/category/draft/page/<int:page>/')
@login_required
@admin_required
def draft(page):
    """Show the Draft projects"""
    return project_index(page, cached_projects.get_all_draft, 'draft',
                     False, True)


@blueprint.route('/category/<string:category>/', defaults={'page': 1})
@blueprint.route('/category/<string:category>/page/<int:page>/')
def project_cat_index(category, page):
    """Show Projects that belong to a given category"""
    return project_index(page, cached_projects.get_all, category, False, True)


@blueprint.route('/new', methods=['GET', 'POST'])
@login_required
def new():
    ensure_authorized_to('create', Project)
    form = ProjectForm(request.form)

    def respond(errors):
        return render_template('projects/new.html',
                               title=gettext("Create a Project"),
                               form=form, errors=errors)

    def _description_from_long_description():
        if form.description.data:
            return form.description.data
        long_desc = form.long_description.data
        html_long_desc = misaka.render(long_desc)[:-1]
        remove_html_tags_regex = re.compile('<[^>]*>')
        blank_space_regex = re.compile('\n')
        text_desc = remove_html_tags_regex.sub("", html_long_desc)[:255]
        if len(text_desc) >= 252:
            text_desc = text_desc[:-3]
            text_desc += "..."
        description = blank_space_regex.sub(" ", text_desc)
        return description if description else " "

    if request.method != 'POST':
        return respond(False)

    if not form.validate():
        flash(gettext('Please correct the errors'), 'error')
        return respond(True)

    info = {}
    category_by_default = cached_cat.get_all()[0]

    project = Project(name=form.name.data,
              short_name=form.short_name.data,
              description=_description_from_long_description(),
              long_description=form.long_description.data,
              owner_id=current_user.id,
              info=info,
              category_id=category_by_default.id)

    project_repo.save(project)

    msg_1 = gettext('Project created!')
    flash('<i class="icon-ok"></i> ' + msg_1, 'success')
    flash('<i class="icon-bullhorn"></i> ' +
          gettext('You can check the ') +
          '<strong><a href="https://docs.pybossa.com">' +
          gettext('Guide and Documentation') +
          '</a></strong> ' +
          gettext('for adding tasks, a thumbnail, using PYBOSSA.JS, etc.'),
          'info')
    auditlogger.add_log_entry(None, project, current_user)

    return redirect(url_for('.update', short_name=project.short_name))


@blueprint.route('/<short_name>/tasks/taskpresentereditor', methods=['GET', 'POST'])
@login_required
def task_presenter_editor(short_name):
    errors = False
    (project, owner, n_tasks, n_task_runs,
     overall_progress,
     last_activity, n_results) = project_by_shortname(short_name)

    title = project_title(project, "Task Presenter Editor")
    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)

    pro = pro_features()

    form = TaskPresenterForm(request.form)
    form.id.data = project.id
    if request.method == 'POST' and form.validate():
        db_project = project_repo.get(project.id)
        old_project = Project(**db_project.dictize())
        old_info = dict(db_project.info)
        old_info['task_presenter'] = form.editor.data
        db_project.info = old_info
        auditlogger.add_log_entry(old_project, db_project, current_user)
        project_repo.update(db_project)
        msg_1 = gettext('Task presenter added!')
        flash('<i class="icon-ok"></i> ' + msg_1, 'success')
        return redirect(url_for('.tasks', short_name=project.short_name))

    # It does not have a validation
    if request.method == 'POST' and not form.validate():  # pragma: no cover
        flash(gettext('Please correct the errors'), 'error')
        errors = True

    if project.info.get('task_presenter'):
        form.editor.data = project.info['task_presenter']
    else:
        if not request.args.get('template'):
            msg_1 = gettext('<strong>Note</strong> You will need to upload the'
                            ' tasks using the')
            msg_2 = gettext('CSV importer')
            msg_3 = gettext(' or download the project bundle and run the'
                            ' <strong>createTasks.py</strong> script in your'
                            ' computer')
            url = '<a href="%s"> %s</a>' % (url_for('project.import_task',
                                                    short_name=project.short_name), msg_2)
            msg = msg_1 + url + msg_3
            flash(msg, 'info')

            wrap = lambda i: "projects/presenters/%s.html" % i
            pres_tmpls = map(wrap, current_app.config.get('PRESENTERS'))

            project = add_custom_contrib_button_to(project, get_user_id_or_ip())
            return render_template(
                'projects/task_presenter_options.html',
                title=title,
                project=project,
                owner=owner,
                overall_progress=overall_progress,
                n_tasks=n_tasks,
                n_task_runs=n_task_runs,
                last_activity=last_activity,
                n_completed_tasks=cached_projects.n_completed_tasks(project.get('id')),
                n_volunteers=cached_projects.n_volunteers(project.get('id')),
                presenters=pres_tmpls,
                pro_features=pro)

        tmpl_uri = "projects/snippets/%s.html" \
            % request.args.get('template')
        tmpl = render_template(tmpl_uri, project=project)
        form.editor.data = tmpl
        msg = 'Your code will be <em>automagically</em> rendered in \
                      the <strong>preview section</strong>. Click in the \
                      preview button!'
        flash(gettext(msg), 'info')
    dict_project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    return render_template('projects/task_presenter_editor.html',
                           title=title,
                           form=form,
                           project=dict_project,
                           owner=owner,
                           overall_progress=overall_progress,
                           n_tasks=n_tasks,
                           n_task_runs=n_task_runs,
                           last_activity=last_activity,
                           n_completed_tasks=cached_projects.n_completed_tasks(project.id),
                           n_volunteers=cached_projects.n_volunteers(project.id),
                           errors=errors,
                           pro_features=pro)


@blueprint.route('/<short_name>/delete', methods=['GET', 'POST'])
@login_required
def delete(short_name):
    (project, owner, n_tasks,
    n_task_runs, overall_progress, last_activity,
    n_results) = project_by_shortname(short_name)
    title = project_title(project, "Delete")
    ensure_authorized_to('read', project)
    ensure_authorized_to('delete', project)
    pro = pro_features()
    if request.method == 'GET':
        return render_template('/projects/delete.html',
                               title=title,
                               project=project,
                               owner=owner,
                               n_tasks=n_tasks,
                               overall_progress=overall_progress,
                               last_activity=last_activity,
                               pro_features=pro)
    project_repo.delete(project)
    auditlogger.add_log_entry(project, None, current_user)
    flash(gettext('Project deleted!'), 'success')
    return redirect(url_for('account.profile', name=current_user.name))


@blueprint.route('/<short_name>/update', methods=['GET', 'POST'])
@login_required
def update(short_name):
    (project, owner, n_tasks,
     n_task_runs, overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    def handle_valid_form(form):
        (project, owner, n_tasks, n_task_runs,
         overall_progress, last_activity,
         n_results) = project_by_shortname(short_name)

        new_project = project_repo.get_by_shortname(short_name)
        old_project = Project(**new_project.dictize())
        old_info = dict(new_project.info)
        old_project.info = old_info
        if form.id.data == new_project.id:
            new_project.name = form.name.data
            new_project.short_name = form.short_name.data
            new_project.description = form.description.data
            new_project.long_description = form.long_description.data
            new_project.webhook = form.webhook.data
            new_project.info = project.info
            new_project.owner_id = project.owner_id
            new_project.allow_anonymous_contributors = form.allow_anonymous_contributors.data
            new_project.category_id = form.category_id.data

        if form.protect.data and form.password.data:
            new_project.set_password(form.password.data)
        if not form.protect.data:
            new_project.set_password("")

        project_repo.update(new_project)
        auditlogger.add_log_entry(old_project, new_project, current_user)
        cached_cat.reset()
        cached_projects.get_project(new_project.short_name)
        flash(gettext('Project updated!'), 'success')
        return redirect(url_for('.details',
                                short_name=new_project.short_name))

    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)

    pro = pro_features()

    title = project_title(project, "Update")
    if request.method == 'GET':
        form = ProjectUpdateForm(obj=project)
        upload_form = AvatarUploadForm()
        categories = project_repo.get_all_categories()
        form.category_id.choices = [(c.id, c.name) for c in categories]
        if project.category_id is None:
            project.category_id = categories[0].id
        form.populate_obj(project)
        form.protect.data = project.needs_password()

    if request.method == 'POST':
        upload_form = AvatarUploadForm()
        form = ProjectUpdateForm(request.form)
        categories = cached_cat.get_all()
        form.category_id.choices = [(c.id, c.name) for c in categories]

        if request.form.get('btn') != 'Upload':
            if form.validate():
                return handle_valid_form(form)
            flash(gettext('Please correct the errors'), 'error')
        else:
            if upload_form.validate_on_submit():
                project = project_repo.get(project.id)
                _file = request.files['avatar']
                coordinates = (upload_form.x1.data, upload_form.y1.data,
                               upload_form.x2.data, upload_form.y2.data)
                prefix = time.time()
                _file.filename = "project_%s_thumbnail_%i.png" % (project.id, prefix)
                container = "user_%s" % current_user.id
                uploader.upload_file(_file,
                                     container=container,
                                     coordinates=coordinates)
                # Delete previous avatar from storage
                if project.info.get('thumbnail'):
                    uploader.delete_file(project.info['thumbnail'], container)
                project.info['thumbnail'] = _file.filename
                project.info['container'] = container
                project_repo.save(project)
                flash(gettext('Your project thumbnail has been updated! It may \
                                  take some minutes to refresh...'), 'success')
            else:
                flash(gettext('You must provide a file to change the avatar'),
                      'error')
            return redirect(url_for('.update', short_name=short_name))

    project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    return render_template('/projects/update.html',
                           form=form,
                           upload_form=upload_form,
                           project=project,
                           owner=owner,
                           n_tasks=n_tasks,
                           overall_progress=overall_progress,
                           n_task_runs=n_task_runs,
                           last_activity=last_activity,
                           n_completed_tasks=cached_projects.n_completed_tasks(project.get('id')),
                           n_volunteers=cached_projects.n_volunteers(project.get('id')),
                           title=title,
                           pro_features=pro)


@blueprint.route('/<short_name>/')
def details(short_name):
    (project, owner, n_tasks, n_task_runs,
     overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    if project.needs_password():
        redirect_to_password = _check_if_redirect_to_password(project)
        if redirect_to_password:
            return redirect_to_password
    else:
        ensure_authorized_to('read', project)

    template = '/projects/project.html'
    pro = pro_features()

    title = project_title(project, None)
    project = add_custom_contrib_button_to(project, get_user_id_or_ip())
    if current_user.is_authenticated() and owner.id == current_user.id:
        project_sanitized = project
        owner_sanitized = cached_users.get_user_summary(owner.name)
    else:   # anonymous or different owner
        if request.headers['Content-Type'] == 'application/json':
            project_sanitized = Project().to_public_json(project)
        else:    # HTML
            project_sanitized = project
        owner_sanitized = cached_users.public_get_user_summary(owner.name)
    template_args = {"project": project_sanitized,
                     "title": title,
                     "owner":  owner_sanitized,
                     "n_tasks": n_tasks,
                     "n_task_runs": n_task_runs,
                     "overall_progress": overall_progress,
                     "last_activity": last_activity,
                     "n_completed_tasks": cached_projects.n_completed_tasks(project.get('id')),
                     "n_volunteers": cached_projects.n_volunteers(project.get('id')),
                     "pro_features": pro}
    if current_app.config.get('CKAN_URL'):
        template_args['ckan_name'] = current_app.config.get('CKAN_NAME')
        template_args['ckan_url'] = current_app.config.get('CKAN_URL')
        template_args['ckan_pkg_name'] = short_name
    response = dict(template=template, **template_args)
    return handle_content_type(response)


@blueprint.route('/<short_name>/settings')
@login_required
def settings(short_name):
    (project, owner, n_tasks, n_task_runs,
     overall_progress, last_activity,
     n_results) = project_by_shortname(short_name)

    title = project_title(project, "Settings")
    ensure_authorized_to('read', project)
    ensure_authorized_to('update', project)
    pro = pro_features()
    project = add_custom_contrib_button_to(project, get_user_id_or_ip())

    if owner.id == current_user.id:
        project_sanitized = project
        owner_sanitized = cached_users.get_user_summary(owner.name)
    else:   # anonymous or different owner
        if request.headers['Content-Type'] == 'application/json':
            project_sanitized = Project().to_public_json(project)
        else:    # HTML
            project_sanitized = project
        owner_sanitized = cached_users.public_get_user_summary(owner.name)

    response = dict(template='/projects/settings.html',
                    project=project_sanitized,
                    owner=owner_sanitized,
                    n_tasks=n_tasks,
                    overall_progress=overall_progress,
                    n_task_runs=n_task_runs,
                    last_activity=last_activity,
                    n_completed_tasks=cached_projects.n_completed_tasks(
                        project.get('id')),
                    n_volunteers=cached_projects.n_volunteers(
{'fraction_non_alphanumeric': 0.09985153764581124, 'fraction_numerical': 0.0025026511134676565, 'mean_word_length': 2.562330009066183, 'pattern_counts': {'":': 34, '<': 73, '<?xml version=': 0, '>': 81, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2939443', 'n_tokens_mistral': 13602, 'n_tokens_neox': 13209, 'n_words': 2671}