<commit_before># -*- coding: utf-8 -*-
from lxml import html
import lxml.etree as le

def get_lead_text (post_text):
    """ Extract the lead text from a post

    The lead text is identified by a <blockquote> with the 'highlights' class"""

    tree = html.fromstring(post_text)

    try:
        # http://stackoverflow.com/questions/29887576/xpath-extract-current-node-content-including-all-child-node
        highlights = tree.xpath('//blockquote[@class="highlights"]')
        lead_text = highlights[0].text + ''.join(le.tostring(node, encoding='unicode') for node in highlights[0])

    except IndexError:
        lead_text = ''

    return lead_text

###############################################################################""

def get_lead_figure (post_text):
    """ Extract the lead figure from a post

    This is similar to get_lead_text except for the lead figure.
    The lead figure is identified by a <figure> with the 'lead-figure' class"""

    tree = html.fromstring(post_text)

    try:
        node = tree.xpath('//figure[@class="lead-figure"]')[0]
        lead_fig = node.text + ''.join(le.tostring(e, encoding='unicode') for e in node)

    except IndexError:
        lead_fig = '<p>ERROR</p>'

    return lead_fig

###############################################################################""<commit_msg>actually add remove_lead_figure jinja filter<commit_after># -*- coding: utf-8 -*-
from lxml import html
import lxml.etree as le

###############################################################################
"""
New filters must be added to TEMPLATE_FILTERS in conf.py
"""
###############################################################################

def get_lead_text (post_text):
    """ Extract the lead text from a post

    The lead text is identified by a <blockquote> with the 'highlights' class"""

    tree = html.fromstring(post_text)

    try:
        # http://stackoverflow.com/questions/29887576/xpath-extract-current-node-content-including-all-child-node
        highlights = tree.xpath('//blockquote[@class="highlights"]')
        lead_text = highlights[0].text + ''.join(le.tostring(node, encoding='unicode') for node in highlights[0])

    except IndexError:
        lead_text = ''

    return lead_text

###############################################################################

def get_lead_figure (post_text):
    """ Extract the lead figure from a post

    This is similar to get_lead_text except for the lead figure.
    The lead figure is identified by a <figure> with the 'lead-figure' class"""

    tree = html.fromstring(post_text)

    try:
        node = tree.xpath('//figure[@class="lead-figure"]')[0]
        lead_fig = node.text + ''.join(le.tostring(e, encoding='unicode') for e in node)

    except IndexError:
        lead_fig = '<p>ERROR</p>'

    return lead_fig

###############################################################################

def remove_lead_figure (post_text):
    """ Remove the lead figure from a post

    This is the opposite of get_lead_figure for when we want to handle them separately.
    The lead figure is identified by a <figure> with the 'lead-figure' class"""

    tree = html.fromstring(post_text)

    try:
        node = tree.xpath('//figure[@class="lead-figure"]')[0]
        node.getparent().remove(node)

    except IndexError:
        return le.tostring(tree, encoding='unicode')

    return le.tostring(tree, encoding='unicode')

###############################################################################
{'fraction_non_alphanumeric': 0.25806451612903225, 'fraction_numerical': 0.007074136955291454, 'mean_word_length': 4.021306818181818, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1726904', 'n_tokens_mistral': 1021, 'n_tokens_neox': 945, 'n_words': 269}