<commit_before>import os
import sys
from collections import Counter

import psutil
from jinja2 import Environment, FileSystemLoader, TemplateNotFound
from markdown import markdown
from markdown.extensions.fenced_code import FencedCodeExtension
from pyatom import AtomFeed
from tornado import web

from summer.ext.search import document_slug
from summer.utils import DotDict
from summer.utils.fn import code_highlighter

DEFAULT_TEMPLATE_PATH = os.path.join(os.getcwd(), 'templates')

fn_markdown = lambda x: markdown(x, extensions=[FencedCodeExtension()])
fn_split = lambda x, y: str(x).split(y)
fn_strip = lambda x: str(x).strip()

def calculate_pagination(res, max_pages=5):
    """ Returns a list of `int`s representing page numbers
        to display at the bottom of a search results page.
    """
    res = DotDict(res)

    # fewer pages than the allowed, return a list
    # with a slot for each
    if res.page_total < max_pages:
        return range(1, res.page_total + 1)

    # towards the end of the pages, we only want to
    # include the `max_pages`
    if res.page_number >= res.page_total - max_pages:
        return range(res.page_total - max_pages, res.page_total + 1)

    # somewhere in the middle
    if max_pages % 2 == 0: # even max pages
        half_page = (max_pages / 2)
        return range(res.page_number - half_page - 1, res.page_number + half_page)

    else: # odd max pages
        half_page = (max_pages // 2)
        return range(res.page_number - half_page, res.page_number + half_page)

def datetime_format(value, format='%m-%d-%Y'):
    return value.strftime(format)

def get_system_load():
    return {
        'cpu': psutil.cpu_percent(interval=None),
        'memory': psutil.virtual_memory(),
        'sys': 'python ' + sys.version.split('(')[0] + sys.platform
    }


class TemplateRender(object):
    """ Class to hold HTML template rendering methods. """

    def render_template(self, name, **kwargs):
        template_path = self.settings.get('template_path', DEFAULT_TEMPLATE_PATH)
        env = Environment(loader=FileSystemLoader([template_path]))

        # filters
        env.filters['calculate_pagination'] = calculate_pagination
        env.filters['datetime_format'] = datetime_format
        env.filters['highlight'] = code_highlighter
        env.filters['markdown'] = fn_markdown
        env.filters['slugify'] = document_slug
        env.filters['split'] = fn_split
        env.filters['strip'] = fn_strip

        try:
            template = env.get_template(name)
        except TemplateNotFound:
            raise TemplateNotFound(name)
        return template.render(kwargs)


class BaseHandler(web.RequestHandler, TemplateRender):

    def __init__(self, application, request, **kwargs):
        super(BaseHandler, self).__init__(application, request, **kwargs)
        self._meta = None

    def get_current_user(self):
        return self.get_secure_cookie('user_id', None)

    def render_html(self, name, **kwargs):
        kwargs.update({
            'csrf': self.xsrf_form_html(),
            'current_user': self.get_current_user(),
            'load': get_system_load(),
            'private': self.meta.private,
            'request': self.request,
            'seo': self.meta.seo,
            'settings': self.settings,
            'this_url': self.this_url,
            'xsrf_token': self.xsrf_token,
            'xsrf_form_html': self.xsrf_form_html })
        self.write(self.render_template(name, **kwargs))

    def get_keywords(self, posts, most_common=20):
        keywords = Counter()
        # extract the most common keywords for the side bar
        for post in posts.results:
            keywords.update([x.strip() for x in post.get('keywords', '').split(',')])
        return keywords.most_common(None)

    def get_topics(self, posts):
        return sorted(list(set(post.get('topic', None) for post in posts.results)))

    @property
    def meta(self):
        if self._meta is None and hasattr(self.application, 'meta'):
            self._meta = DotDict(self.application.meta)
        return self._meta

    @property
    def this_url(self):
        return '{}://{}{}'.format(
            self.meta.settings.protocol, self.meta.settings.domain, self.request.uri)

    @staticmethod
    def generate_feed(posts, subtitle='', url="https://sowingseasons.com/feed.atom"):
        feed = AtomFeed(
            title="SowingSeasons",
            title_type="text",
            subtitle="takes awhile to grow anything. %s" % subtitle,
            subtitle_type="text",
            feed_url=url,
            url="https://sowingseasons.com",
            author="<NAME>",
            icon="sowingseasons.com/static/img/ico_black.png",
            logo="sowingseasons.com/static/img/logo.png",
            rights="MIT LICENSE",
            rights_type="text",
            generator=("PyAtom", "https://github.com/sramana/pyatom", "1.4")
        )

        for post in posts.results:
            post = DotDict(post)

            feed.add(
                title=post.title,
                title_type="text",
                content=fn_markdown(post.content),
                content_type="html",
                summary=post.summary,
                summary_type="text",
                url='https://sowingseasons.com' + document_slug(post),
                updated=post.modified,
                author="<NAME>",
                published=post.modified,
                rights="MIT LICENSE",
                rights_type="text"
            )

        return feed.to_string()<commit_msg>Day 2, Refactor `BaseHandler`<commit_after>import os
import sys
import logging
from collections import Counter

import psutil
from jinja2 import Environment, FileSystemLoader, TemplateNotFound
from markdown import markdown
from markdown.extensions.fenced_code import FencedCodeExtension
from pyatom import AtomFeed
from tornado import web

from summer.ext.search import document_slug
from summer.utils import DotDict
from summer.utils.fn import code_highlighter

logger = logging.getLogger(__name__)

DEFAULT_TEMPLATE_PATH = os.path.join(os.getcwd(), 'templates')

fn_markdown = lambda x: markdown(x, extensions=[FencedCodeExtension()])
fn_split = lambda x, y: str(x).split(y)
fn_strip = lambda x: str(x).strip()

def calculate_pagination(res, max_pages=5):
    """ Returns a list of `int`s representing page numbers
        to display at the bottom of a search results page.
    """
    res = DotDict(res)

    # fewer pages than the allowed, return a list
    # with a slot for each
    if res.page_total < max_pages:
        return range(1, res.page_total + 1)

    # towards the end of the pages, we only want to
    # include the `max_pages`
    if res.page_number >= res.page_total - max_pages:
        return range(res.page_total - max_pages, res.page_total + 1)

    # somewhere in the middle
    if max_pages % 2 == 0: # even max pages
        half_page = (max_pages / 2)
        return range(res.page_number - half_page - 1, res.page_number + half_page)

    else: # odd max pages
        half_page = (max_pages // 2)
        return range(res.page_number - half_page, res.page_number + half_page)

def datetime_format(value, format='%m-%d-%Y'):
    return value.strftime(format)

def get_system_load():
    return {
        'cpu': psutil.cpu_percent(interval=None),
        'memory': psutil.virtual_memory(),
        'sys': 'python ' + sys.version.split('(')[0] + sys.platform
    }


class TemplateRender(object):
    """ Class to hold HTML template rendering methods. """

    def render_template(self, name, **kwargs):
        template_path = self.settings.get('template_path', DEFAULT_TEMPLATE_PATH)
        env = Environment(loader=FileSystemLoader([template_path]))

        # filters
        custom_filters = [
            ('calculate_pagination', calculate_pagination),
            ('datetime_format', datetime_format),
            ('highlight', code_highlighter),
            ('markdown', fn_markdown),
            ('slugify', document_slug),
            ('split', fn_split),
            ('strip', fn_strip)
        ]

        for fname, fn in custom_filters:
            env.filters[fname] = fn

        try:
            template = env.get_template(name)
        except TemplateNotFound:
            logger.error('Could not find template, %s' % name)
            raise TemplateNotFound(name)
        else:
            logger.info('render_template(%s)' % name)

        return template.render(kwargs)


class BaseHandler(web.RequestHandler, TemplateRender):

    def __init__(self, application, request, **kwargs):
        super(BaseHandler, self).__init__(application, request, **kwargs)
        self._meta = None

    def get_current_user(self):
        return self.get_secure_cookie('user_id', None)

    def render_html(self, name, **kwargs):
        kwargs.update({
            'csrf': self.xsrf_form_html(),
            'current_user': self.get_current_user(),
            'load': get_system_load(),
            'private': self.meta.private,
            'request': self.request,
            'seo': self.meta.seo,
            'settings': self.settings,
            'this_url': self.this_url,
            'xsrf_token': self.xsrf_token,
            'xsrf_form_html': self.xsrf_form_html })
        self.write(self.render_template(name, **kwargs))

    @property
    def meta(self):
        if self._meta is None and hasattr(self.application, 'meta'):
            self._meta = DotDict(self.application.meta)
        return self._meta

    @property
    def this_url(self):
        return '{}://{}{}'.format(
            self.meta.settings.protocol, self.meta.settings.domain, self.request.uri)

    @staticmethod
    def get_keywords(posts, most_common=20):
        keywords = Counter()
        # extract the most common keywords for the side bar
        for post in posts.results:
            keywords.update([x.strip() for x in post.get('keywords', '').split(',')])
        return keywords.most_common(None)

    @staticmethod
    def get_topics(posts):
        return sorted(list(set(post.get('topic', None) for post in posts.results)))

    @staticmethod
    def generate_feed(posts, subtitle='', url="https://sowingseasons.com/feed.atom"):
        logger.info('generate_feed(%s)' % url)

        feed = AtomFeed(
            title="SowingSeasons",
            title_type="text",
            subtitle="takes awhile to grow anything. %s" % subtitle,
            subtitle_type="text",
            feed_url=url,
            url="https://sowingseasons.com",
            author="<NAME>",
            icon="/static/img/ico_black.png",
            logo="/static/img/logo.png",
            rights="MIT LICENSE",
            rights_type="text",
            generator=("PyAtom", "https://github.com/sramana/pyatom", "1.4")
        )

        for post in posts.results:
            post = DotDict(post)

            feed.add(
                title=post.title,
                title_type="text",
                content=fn_markdown(post.content),
                content_type="html",
                summary=post.summary,
                summary_type="text",
                url='https://sowingseasons.com' + document_slug(post),
                updated=post.modified,
                author="<NAME>",
                published=post.modified,
                rights="MIT LICENSE",
                rights_type="text"
            )

        return feed.to_string()
{'fraction_non_alphanumeric': 0.11549886423204613, 'fraction_numerical': 0.0027083697361523678, 'mean_word_length': 2.4814476885644767, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3715972', 'n_tokens_mistral': 3356, 'n_tokens_neox': 3249, 'n_words': 769}