<commit_before>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError
import lxml.html

import logging
logger = logging.getLogger('ti.providers.webpage')

class Webpage(Provider):  

    example_id = ("url", "http://total-impact.org/")

    biblio_url_template = "%s"
    provenance_url_template = "%s"
    descr = "Information scraped from webpages by total-impact"
    url = "http://total-impact.org"


    def __init__(self):
        super(Webpage, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("url" == namespace)


    # use lxml because is html

    def _extract_biblio(self, page, id=None):
        #dict_of_keylists = {
        #    'title' : ['html', 'head', 'title'],
        #    'h1' : ['h1']
        #}

        biblio_dict = {}
        parsed_html = lxml.html.document_fromstring(page.encode("utf-8"))
        
        try:
            biblio_dict["title"] = parsed_html.find(".//title").text
        except AttributeError:
            pass

        try:
            biblio_dict["h1"] = parsed_html.find(".//h1").text
        except AttributeError:
            pass

        return biblio_dict    <commit_msg>only include webpage metrics if they are not empty<commit_after>from totalimpact.providers import provider
from totalimpact.providers.provider import Provider, ProviderContentMalformedError
import lxml.html

import logging
logger = logging.getLogger('ti.providers.webpage')

class Webpage(Provider):  

    example_id = ("url", "http://total-impact.org/")

    biblio_url_template = "%s"
    provenance_url_template = "%s"
    descr = "Information scraped from webpages by total-impact"
    url = "http://total-impact.org"


    def __init__(self):
        super(Webpage, self).__init__()

    def is_relevant_alias(self, alias):
        (namespace, nid) = alias
        return("url" == namespace)


    # use lxml because is html

    def _extract_biblio(self, page, id=None):
        #dict_of_keylists = {
        #    'title' : ['html', 'head', 'title'],
        #    'h1' : ['h1']
        #}

        biblio_dict = {}
        parsed_html = lxml.html.document_fromstring(page.encode("utf-8"))
        
        try:
            response = parsed_html.find(".//h1").text
            if response:
                biblio_dict["title"] = response
        except AttributeError:
            pass

        try:
            response = parsed_html.find(".//h1").text
            if response:
                biblio_dict["h1"] = response
        except AttributeError:
            pass

        return biblio_dict    
{'fraction_non_alphanumeric': 0.11936936936936937, 'fraction_numerical': 0.004129129129129129, 'mean_word_length': 2.352201257861635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7567586', 'n_tokens_mistral': 833, 'n_tokens_neox': 785, 'n_words': 172}