<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import httpkie
import dhtmlparser

from structures import Author
from structures import Publication
# from ..structures import Author
# from ..structures import Publication


# Variables ===================================================================
URL = "http://shop.ben.cz/"
URL += r"Produkty.aspx?lang=cz&nak=BEN+-+technick%u00e1+literatura"
DOWNER = httpkie.Downloader()
DOWNER.cookies = {
    "shop.ben.cz": {
        "pageSize": "100",
        "viewProductSize": "tabulka"
    }
}


# Functions & objects =========================================================
def _get_last_td(el):
    """
    Return last <td> found in `el` DOM.

    Args:
        el (obj): :class:`dhtmlparser.HTMLElement` instance.

    Returns:
        obj: HTMLElement instance if found, or None if there are no <td> tags.
    """
    if not el:
        return None

    if type(el) in [list, tuple, set]:
        el = el[0]

    last = el.find("td")

    if not last:
        return None

    return last[-1]


def _get_td_or_none(details, ID):
    """
    Get <tr> tag with given `ID` and return content of the last <td> tag from
    <tr> root.

    Args:
        details (obj): :class:`dhtmlparser.HTMLElement` instance.
        ID (str): id property of the <tr> tag.

    Returns:
        str: Content of the last <td> as strign.
    """
    content = details.find("tr", {"id": ID})
    content = _get_last_td(content)

    # if content is None, return it
    if not content:
        return None

    content = content.getContent().strip()

    # if content is blank string, return None
    if not content:
        return None

    return content


# Parsers =====================================================================
def _parse_title(dom, details):
    title = details.find("h1")

    # if the header is missing, try to parse title from the <title> tag
    if not title:
        title = dom.find("title")
        assert title, "Can't find <title> tag!"

        return title[0].getContent().split("|")[0].strip()

    return title[0].getContent().strip()


def _parse_authors(details):
    authors = details.find(
        "tr",
        {"id": "ctl00_ContentPlaceHolder1_tblRowAutor"}
    )

    if not authors:
        return []  # book with unspecified authors

    # parse authors from HTML and convert them to Author objects
    author_list = []
    for author in authors[0].find("a"):
        author_obj = Author(author.getContent())

        if "href" in author.params:
            author_obj.URL = author.params["href"]

        author_list.append(author_obj)

    return author_list


def _parse_publisher(details):
    publisher = details.find(
        "td",
        {"id": "ctl00_ContentPlaceHolder1_TableCell3"}
    )

    # publisher is not specified
    if not publisher:
        return None

    publisher = dhtmlparser.removeTags(publisher[0]).strip()

    # return None instead of blank string
    if not publisher:
        return None

    return publisher

<commit_msg>#13: Fixed bug in _parse_publisher().<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import httpkie
import dhtmlparser

from structures import Author
from structures import Publication
# from ..structures import Author
# from ..structures import Publication


# Variables ===================================================================
URL = "http://shop.ben.cz/"
URL += r"Produkty.aspx?lang=cz&nak=BEN+-+technick%u00e1+literatura"
DOWNER = httpkie.Downloader()
DOWNER.cookies = {
    "shop.ben.cz": {
        "pageSize": "100",
        "viewProductSize": "tabulka"
    }
}


# Functions & objects =========================================================
def _get_last_td(el):
    """
    Return last <td> found in `el` DOM.

    Args:
        el (obj): :class:`dhtmlparser.HTMLElement` instance.

    Returns:
        obj: HTMLElement instance if found, or None if there are no <td> tags.
    """
    if not el:
        return None

    if type(el) in [list, tuple, set]:
        el = el[0]

    last = el.find("td")

    if not last:
        return None

    return last[-1]


def _get_td_or_none(details, ID):
    """
    Get <tr> tag with given `ID` and return content of the last <td> tag from
    <tr> root.

    Args:
        details (obj): :class:`dhtmlparser.HTMLElement` instance.
        ID (str): id property of the <tr> tag.

    Returns:
        str: Content of the last <td> as strign.
    """
    content = details.find("tr", {"id": ID})
    content = _get_last_td(content)

    # if content is None, return it
    if not content:
        return None

    content = content.getContent().strip()

    # if content is blank string, return None
    if not content:
        return None

    return content


# Parsers =====================================================================
def _parse_title(dom, details):
    title = details.find("h1")

    # if the header is missing, try to parse title from the <title> tag
    if not title:
        title = dom.find("title")
        assert title, "Can't find <title> tag!"

        return title[0].getContent().split("|")[0].strip()

    return title[0].getContent().strip()


def _parse_authors(details):
    authors = details.find(
        "tr",
        {"id": "ctl00_ContentPlaceHolder1_tblRowAutor"}
    )

    if not authors:
        return []  # book with unspecified authors

    # parse authors from HTML and convert them to Author objects
    author_list = []
    for author in authors[0].find("a"):
        author_obj = Author(author.getContent())

        if "href" in author.params:
            author_obj.URL = author.params["href"]

        author_list.append(author_obj)

    return author_list


def _parse_publisher(details):
    publisher = _get_td_or_none(
        details,
        "ctl00_ContentPlaceHolder1_tblRowNakladatel"
    )

    # publisher is not specified
    if not publisher:
        return None

    publisher = dhtmlparser.removeTags(publisher).strip()

    # return None instead of blank string
    if not publisher:
        return None

    return publisher


{'fraction_non_alphanumeric': 0.18717586044318718, 'fraction_numerical': 0.007543611504007544, 'mean_word_length': 2.8947368421052633, 'pattern_counts': {'":': 11, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5443481', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1809, 'n_words': 555}