<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import logging
import lxml.etree, lxml.html
from datetime import datetime
from dateutil.parser import parse
from dateutil.tz import tzoffset
from collections import ChainMap

logger = logging.getLogger(__name__)

class HtmlMeta(object):
    """Extract metadata from html.
    Needs work, e.g. handling multiple @property=author tags,
    detect if author content is a social media destination.
    """
    def __init__(self, html, encoding="UTF-8", tree=None):
        self.html = html or None
        if tree is not None:
            # reuse tree already parsed
            self.document = tree
        else:
            self.parser = lxml.html.HTMLParser(encoding=encoding)
            self.document = lxml.html.fromstring(html, parser=self.parser)
        self._jsonld_xpath = lxml.etree.XPath('descendant-or-self::script[@type="application/ld+json"]')
        self._metatag_xpath = lxml.etree.XPath("//meta")
        self._links_xpath = lxml.etree.XPath("//link")

        self.links = {}
        self.jsonld = {}
        self.metatags = {}

    @property
    def title(self):
        return self.jsonld.get("headline") \
            or self.metatags.get("og:title") \
                or self.extract_title()

    @property
    def description(self):
        return self.metatags.get("og:description") \
            or self.metatags.get("description", "").strip()

    @property
    def canonical_url(self):
        return self.links.get("canonical")

    @property
    def image(self):
        return self.metatags.get("og:image") \
            or self.jsonld.get("thumbnailUrl")

    def _extract_ld_authors(self):
        # extract from jsonld
        ld_authors = self.jsonld.get("author", {})
        # sanitize ld structure
        if type(ld_authors) == str:
            ld_authors = {"name": ld_authors}
        ld_authors = [a["name"] for a in ld_authors] if type(ld_authors) == list else ld_authors.get("name", False)
        return ld_authors

    @property
    def authors(self):
        # get a value from trove
        authors = self._extract_ld_authors() \
            or self.metatags.get("author") \
                or self.metatags.get("article:author") \
                    or self.metatags.get("dcterms.creator") \
                        or self.metatags.get("article:authorName") \
                            or self.metatags.get("citation_author") \
                                or self.jsonld.get("authors") # intercept

        if authors:
            # ensure list
            if type(authors) != list:
                authors = [authors]
            # strip links
            authors = [a for a in authors if a.startswith("http") == False]

        if not authors:
            # washingtonpost
            xauthors = self.document.xpath("(//span[@itemprop='author'])[1]//span[@itemprop='name']/text()")
            if xauthors:
                authors = xauthors<commit_msg>edge case ld+author present but unparseable<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import logging
import lxml.etree, lxml.html
from datetime import datetime
from dateutil.parser import parse
from dateutil.tz import tzoffset
from collections import ChainMap

logger = logging.getLogger(__name__)

class HtmlMeta(object):
    """Extract metadata from html.
    Needs work, e.g. handling multiple @property=author tags,
    detect if author content is a social media destination.
    """
    def __init__(self, html, encoding="UTF-8", tree=None):
        self.html = html or None
        if tree is not None:
            # reuse tree already parsed
            self.document = tree
        else:
            self.parser = lxml.html.HTMLParser(encoding=encoding)
            self.document = lxml.html.fromstring(html, parser=self.parser)
        self._jsonld_xpath = lxml.etree.XPath('descendant-or-self::script[@type="application/ld+json"]')
        self._metatag_xpath = lxml.etree.XPath("//meta")
        self._links_xpath = lxml.etree.XPath("//link")

        self.links = {}
        self.jsonld = {}
        self.metatags = {}

    @property
    def title(self):
        return self.jsonld.get("headline") \
            or self.metatags.get("og:title") \
                or self.extract_title()

    @property
    def description(self):
        return self.metatags.get("og:description") \
            or self.metatags.get("description", "").strip()

    @property
    def canonical_url(self):
        return self.links.get("canonical")

    @property
    def image(self):
        return self.metatags.get("og:image") \
            or self.jsonld.get("thumbnailUrl")

    def _extract_ld_authors(self):
        # extract from jsonld
        ld_authors = self.jsonld.get("author", {})

        # Return if unparseable
        if not ld_authors:
            return None

        # sanitize ld structure
        if type(ld_authors) == str:
            ld_authors = {"name": ld_authors}

        ld_authors = [a["name"] for a in ld_authors] if type(ld_authors) == list else ld_authors.get("name", False)
        return ld_authors

    @property
    def authors(self):
        # get a value from trove
        authors = self._extract_ld_authors() \
            or self.metatags.get("author") \
                or self.metatags.get("article:author") \
                    or self.metatags.get("dcterms.creator") \
                        or self.metatags.get("article:authorName") \
                            or self.metatags.get("citation_author") \
                                or self.jsonld.get("authors") # intercept

        if authors:
            # ensure list
            if type(authors) != list:
                authors = [authors]
            # strip links
            authors = [a for a in authors if a.startswith("http") == False]

        if not authors:
            # washingtonpost
            xauthors = self.document.xpath("(//span[@itemprop='author'])[1]//span[@itemprop='name']/text()")
            if xauthors:
                authors = xauthors
{'fraction_non_alphanumeric': 0.11752136752136752, 'fraction_numerical': 0.0013149243918474688, 'mean_word_length': 2.0888324873096447, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6503858', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1731, 'n_words': 408}