<commit_before># -*- coding: utf-8 -*-
#
# Copyright(c) 2010 poweredsites.org
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging
from tornado.web import asynchronous
from tornado import escape
from tornado.httpclient import AsyncHTTPClient
from tornado.web import UIModule

#import chardet
from BeautifulSoup import BeautifulSoup

from poweredsites.libs.handler import BaseHandler
from poweredsites.forms.submit import ProjectForm, ProjectPreForm
from poweredsites.libs.decorators import cache, authenticated
from poweredsites.libs import const


class ProjectBaseHandler(BaseHandler):
    @property
    def categories(self):
        return self.db.query("select * from category order by id ASC")


class ProjectMainHandler(ProjectBaseHandler):
    _category_top_query = "select * from ("\
                "(select count(site_id) as c, project_id from project_sites "\
                "group by project_id) as ps "\
                "left join (select id, subdomain, category_id from project) "\
                "as p on ps.project_id = p.id) where p.category_id=%s order by ps.c DESC limit 0, 5"

    _category_count_query = "select sum(ps.c) as c from (select count(site_id) as c, project_id from "\
                "project_sites group by project_id) as ps "\
                "left join (select id, category_id from project) as p on ps.project_id = p.id "\
                "where p.category_id = %s"

    @cache.page()
    def get(self):
        top_projects = self.db.query("select * from ("
                "(select count(site_id) as c, project_id from project_sites "
                "group by project_id order by c DESC limit 0, 10) as ps "
                "left join (select id, subdomain from project) as p on ps.project_id = p.id)")
        top_projects_count = self._get_count("select count(*) as c from project_sites")

        top_frameworks = self.db.query(self._category_top_query, const.Category.FRAMEWORK)
        top_frameworks_count = self._get_count(self._category_count_query % const.Category.FRAMEWORK)

        top_projects = escape.json_encode(top_projects)
        top_frameworks = escape.json_encode(top_frameworks)

        self.render("project/main.html", top_projects_count=top_projects_count,
                    top_projects=top_projects,
                    top_frameworks=top_frameworks,
                    top_frameworks_count=top_frameworks_count,
                    )

    def _get_count(self, query):
        count = self.db.get(query)
        if count:
            count = count.c
        else:
            count = 0

        return count


class SubmitProjectHandler(ProjectBaseHandler):
    _submit_template = "project/submit.html"

    @authenticated
    @asynchronous
    def get(self):
        self._context.title = "Submit project"

        website = self.request.arguments.get("url", [""])[0]
        if website:
            http = AsyncHTTPClient()
            http.fetch(escape.url_unescape(website), self._on_fetch)
        else:
            self._context.metainfos = {}
            self.render(self._submit_template)

    def _on_fetch(self, response):
        metainfos = {"website":response.request.url}

        if response.code == 200:
            #encoding = chardet.detect(response.body)["encoding"]
            try:
                bs = BeautifulSoup(response.body)
                metas = bs.findAll("meta")
                metainfos["sitename"] = bs.find("title")
                if metainfos["sitename"] is None:
                    metainfos["sitename"] = ""
                else:
                    metainfos["sitename"] = metainfos["sitename"].contents[0]

                for meta in metas:
                    name = meta.get("name")
                    if name is not None:
                        content = meta.get("content")
                        if content is None:
                            content = ""
                        #metainfos[name] = unicode(content, encoding).encode("utf-8")
                        metainfos[name] = content
            except Exception, e:
                logging.error(str(e))

        self._context.metainfos = metainfos
        self.render(self._submit_template)

    @authenticated
    def post(self):
        fm = ProjectForm(self)
        self._context.metainfos = {}

        if fm.validate():
            self.redirect("http://" + fm._values["subdomain"].lower() + ".poweredsites.org/")
        else:
            fm.render(self._submit_template)


class SubmitProjectPreHandler(ProjectBaseHandler):
    @authenticated
    def get(self):
        self._context.title = "Submit project"
        self.render("project/submit_pre.html")

    @authenticated
    def post(self):
        fm = ProjectPreForm(self)

        if fm.validate():
            self.redirect("/submit/project?url=%s" % escape.url_escape(fm._values["website"]))
        else:
            fm.render("project/submit_pre.html")


class ProjectIndexHandler(ProjectBaseHandler):
    _order_by = "id DESC"
    _condition = ""
    _handler_template = "project/index.html"
    _ws_count_query = "select count(*) as c from project_sites where project_id = %s"
    _context_title = " - latest sites"

    @property
    def cache_condition(self):
        subdomain = self.request.host.split(".")[0]
        if not subdomain.islower():
            self.redirect(self.request.host.lower())
        else:
            current_project = self.db.get("select * from project where subdomain = %s", subdomain)
            if current_project:
                condition = "select id from project_sites where project_id = %s order by id DESC" % current_project.id
                return str(condition)

        return ""

    @cache.page(1200, condition="select count(*) from site")
    def get(self):
        subdomain = self.request.host.split(".")[0]
        if not subdomain.islower():
            self.redirect(self.request.host.lower())
        else:
            current_project = self.db.get("select * from project where subdomain = %s", subdomain)
            if current_project:
                self._context.ws_count_query = str(self._ws_count_query % current_project.id)
                self._context.ws_query = "select site.*, user.username, user.openid_name "\
                        "from site, user, project_sites where site.user_id = user.id "\
                        "and project_sites.project_id = %s "\
                        "and site.id = project_sites.site_id "\
                        "%s order by %s" % \
                        (current_project.id, self._condition, self._order_by)
                self._context.ws_query = str(self._context.ws_query)
                self._context.page = self.get_argument("page", 1)<commit_msg>use condition args rather than cache_condition for a project page<commit_after># -*- coding: utf-8 -*-
#
# Copyright(c) 2010 poweredsites.org
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import logging
from tornado.web import asynchronous
from tornado import escape
from tornado.httpclient import AsyncHTTPClient
from tornado.web import UIModule

#import chardet
from BeautifulSoup import BeautifulSoup

from poweredsites.libs.handler import BaseHandler
from poweredsites.forms.submit import ProjectForm, ProjectPreForm
from poweredsites.libs.decorators import cache, authenticated
from poweredsites.libs import const


class ProjectBaseHandler(BaseHandler):
    @property
    def categories(self):
        return self.db.query("select * from category order by id ASC")


class ProjectMainHandler(ProjectBaseHandler):
    _category_top_query = "select * from ("\
                "(select count(site_id) as c, project_id from project_sites "\
                "group by project_id) as ps "\
                "left join (select id, subdomain, category_id from project) "\
                "as p on ps.project_id = p.id) where p.category_id=%s order by ps.c DESC limit 0, 5"

    _category_count_query = "select sum(ps.c) as c from (select count(site_id) as c, project_id from "\
                "project_sites group by project_id) as ps "\
                "left join (select id, category_id from project) as p on ps.project_id = p.id "\
                "where p.category_id = %s"

    @cache.page()
    def get(self):
        top_projects = self.db.query("select * from ("
                "(select count(site_id) as c, project_id from project_sites "
                "group by project_id order by c DESC limit 0, 10) as ps "
                "left join (select id, subdomain from project) as p on ps.project_id = p.id)")
        top_projects_count = self._get_count("select count(*) as c from project_sites")

        top_frameworks = self.db.query(self._category_top_query, const.Category.FRAMEWORK)
        top_frameworks_count = self._get_count(self._category_count_query % const.Category.FRAMEWORK)

        top_projects = escape.json_encode(top_projects)
        top_frameworks = escape.json_encode(top_frameworks)

        self.render("project/main.html", top_projects_count=top_projects_count,
                    top_projects=top_projects,
                    top_frameworks=top_frameworks,
                    top_frameworks_count=top_frameworks_count,
                    )

    def _get_count(self, query):
        count = self.db.get(query)
        if count:
            count = count.c
        else:
            count = 0

        return count


class SubmitProjectHandler(ProjectBaseHandler):
    _submit_template = "project/submit.html"

    @authenticated
    @asynchronous
    def get(self):
        self._context.title = "Submit project"

        website = self.request.arguments.get("url", [""])[0]
        if website:
            http = AsyncHTTPClient()
            http.fetch(escape.url_unescape(website), self._on_fetch)
        else:
            self._context.metainfos = {}
            self.render(self._submit_template)

    def _on_fetch(self, response):
        metainfos = {"website":response.request.url}

        if response.code == 200:
            #encoding = chardet.detect(response.body)["encoding"]
            try:
                bs = BeautifulSoup(response.body)
                metas = bs.findAll("meta")
                metainfos["sitename"] = bs.find("title")
                if metainfos["sitename"] is None:
                    metainfos["sitename"] = ""
                else:
                    metainfos["sitename"] = metainfos["sitename"].contents[0]

                for meta in metas:
                    name = meta.get("name")
                    if name is not None:
                        content = meta.get("content")
                        if content is None:
                            content = ""
                        #metainfos[name] = unicode(content, encoding).encode("utf-8")
                        metainfos[name] = content
            except Exception, e:
                logging.error(str(e))

        self._context.metainfos = metainfos
        self.render(self._submit_template)

    @authenticated
    def post(self):
        fm = ProjectForm(self)
        self._context.metainfos = {}

        if fm.validate():
            self.redirect("http://" + fm._values["subdomain"].lower() + ".poweredsites.org/")
        else:
            fm.render(self._submit_template)


class SubmitProjectPreHandler(ProjectBaseHandler):
    @authenticated
    def get(self):
        self._context.title = "Submit project"
        self.render("project/submit_pre.html")

    @authenticated
    def post(self):
        fm = ProjectPreForm(self)

        if fm.validate():
            self.redirect("/submit/project?url=%s" % escape.url_escape(fm._values["website"]))
        else:
            fm.render("project/submit_pre.html")


class ProjectIndexHandler(ProjectBaseHandler):
    _order_by = "id DESC"
    _condition = ""
    _handler_template = "project/index.html"
    _ws_count_query = "select count(*) as c from project_sites where project_id = %s"
    _context_title = " - latest sites"

    @cache.page(1200, condition="select count(*) from site")
    def get(self):
        subdomain = self.request.host.split(".")[0]
        if not subdomain.islower():
            self.redirect(self.request.host.lower())
        else:
            current_project = self.db.get("select * from project where subdomain = %s", subdomain)
            if current_project:
                self._context.ws_count_query = str(self._ws_count_query % current_project.id)
                self._context.ws_query = "select site.*, user.username, user.openid_name "\
                        "from site, user, project_sites where site.user_id = user.id "\
                        "and project_sites.project_id = %s "\
                        "and site.id = project_sites.site_id "\
                        "%s order by %s" % \
                        (current_project.id, self._condition, self._order_by)
                self._context.ws_query = str(self._context.ws_query)
                self._context.page = self.get_argument("page", 1)
{'fraction_non_alphanumeric': 0.09894270610087155, 'fraction_numerical': 0.003929132733247607, 'mean_word_length': 2.2450162262401485, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3723970', 'n_tokens_mistral': 4047, 'n_tokens_neox': 3794, 'n_words': 990}