<commit_before>import lxml.html
import feedparser
from django.core.cache import cache

OPENHATCH_BLOG_FEED_URL='http://openhatch.org/blog/feed/atom/'

def summary2html(html_string):
    decoded = lxml.html.fragment_fromstring('<div>' + html_string + '</div>')
    text = decoded.text_content()
    return unicode(text)

def _blog_entries():
    parsed = feedparser.parse(OPENHATCH_BLOG_FEED_URL)
    for entry in parsed['entries']:
        entry['unicode_text'] = summary2html(entry['summary'])
    return parsed['entries']

def cached_blog_entries():
    key_name = 'blog_entries'
    entries = cache.get(key_name)
    if entries is None:
        entries = _blog_entries()
        # cache it for 30 minutes
        cache.set(key_name, entries, 30 * 60)
    return entries
<commit_msg>Landing page: Now special characters when used in blog posts show up correctly.<commit_after>import lxml.html
import feedparser
from django.core.cache import cache

OPENHATCH_BLOG_FEED_URL='http://openhatch.org/blog/feed/atom/'

def summary2html(html_string):
    decoded = lxml.html.fragment_fromstring('<div>' + html_string + '</div>')
    text = decoded.text_content()
    return unicode(text)

def _blog_entries():
    parsed = feedparser.parse(OPENHATCH_BLOG_FEED_URL)
    for entry in parsed['entries']:
        entry['unicode_text'] = summary2html(entry['summary'])
        entry['title'] = summary2html(entry['title'])
    return parsed['entries']

def cached_blog_entries():
    key_name = 'blog_entries'
    entries = cache.get(key_name)
    if entries is None:
        entries = _blog_entries()
        # cache it for 30 minutes
        cache.set(key_name, entries, 30 * 60)
    return entries

{'fraction_non_alphanumeric': 0.11420118343195267, 'fraction_numerical': 0.010059171597633136, 'mean_word_length': 3.9589442815249267, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4295798', 'n_tokens_mistral': 544, 'n_tokens_neox': 527, 'n_words': 122}