<commit_before>#!/usr/bin/env python
from __future__ import print_function

import sys
import argparse

import requests
from lxml import etree

ARCHIVEJS = 'http://archive.cnx.org/contents/{}.json'
ARCHIVEHTML = 'http://archive.cnx.org/contents/{}.html'
NS = {'x': 'http://www.w3.org/1999/xhtml'}
HTMLWRAPPER = """<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>{title}</title>
<link href="styles.css" rel="stylesheet" type="text/css"/>
</head>
</html>
"""

parts = ['page', 'chapter', 'unit', 'book', 'series']


def debug(*args, **kwargs):
    if verbose:
        print(*args, file=sys.stderr, **kwargs)


def main(code, html_out=sys.stdout):
    """Generate complete book HTML."""

    res = requests.get(ARCHIVEJS.format(code))
    b_json = res.json()
    html = etree.fromstring(HTMLWRAPPER.format(title=b_json['title']))
    book_elem = etree.SubElement(html, 'body', attrib={'data-type': 'book'})

    html_nest([b_json['tree']], book_elem)

    print(etree.tostring(html), file=html_out)


def html_nest(tree, parent):
    """Recursively construct HTML nested div version of book tree."""
    for node in tree:
        div_elem = etree.SubElement(parent, 'div')
        if node['id'] != 'subcol':
            page_nodes(node['id'], div_elem)
            mytype = parts.index(div_elem.get('data-type'))
            if parent.get('data-type'):
                parenttype = parts.index(parent.get('data-type'))
                if parenttype <= mytype:
                    parent.set('data-type', parts[mytype + 1])
            else:
                parent.set('data-type', parts[mytype + 1])

        title_xpath = etree.XPath("//x:div[@data-type='document-title']",
                                  namespaces=NS)
        try:
            title_elem = title_xpath(div_elem)[0]
        except IndexError:
            title_elem = etree.SubElement(div_elem, 'div',
                                          attrib={'data-type':
                                                  'document-title'})
        title_elem.text = node['title']
        debug(node['title'])
        if 'contents' in node:
            elem = etree.SubElement(div_elem, 'div')
            html_nest(node['contents'], elem)


def page_nodes(page_id, elem):
    """Fetch page return body wrapped in provided element."""

    debug(page_id)
    res = requests.get(ARCHIVEHTML.format(page_id))
    xpath = etree.XPath('//x:body', namespaces=NS)

    body = xpath(etree.fromstring(res.content))[0]

    elem.set('data-type', 'page')
    for c in body.iterchildren():
        elem.append(c)

    return elem


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description="Assemble complete book "
                                                 "as single HTML file")
    parser.add_argument("bookid", help="Identifier of book: "
                        "<uuid|shortId>[@ver]")
    parser.add_argument("html_out", nargs="?",
                        type=argparse.FileType('w'),
                        help="assembled HTML file output (default stdout)",
                        default=sys.stdout)
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Send debugging info to stderr')
    args = parser.parse_args()
    verbose = args.verbose
    main(args.bookid, args.html_out)<commit_msg>Add sample-book option. Fix part naming and title override<commit_after>#!/usr/bin/env python
from __future__ import print_function

import sys
import argparse

import requests
from lxml import etree

ARCHIVEJS = 'http://archive.cnx.org/contents/{}.json'
ARCHIVEHTML = 'http://archive.cnx.org/contents/{}.html'
NS = {'x': 'http://www.w3.org/1999/xhtml'}
HTMLWRAPPER = """<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>{title}</title>
<link href="styles.css" rel="stylesheet" type="text/css"/>
</head>
</html>
"""

title_xpath = etree.XPath("./x:div[@data-type='document-title']",
                          namespaces=NS)

parts = ['page', 'chapter', 'unit', 'book', 'series']
partcount = {}.fromkeys(parts, 0)


def debug(*args, **kwargs):
    if verbose:
        print(*args, file=sys.stderr, **kwargs)


def main(code, html_out=sys.stdout):
    """Generate complete book HTML."""

    res = requests.get(ARCHIVEJS.format(code))
    b_json = res.json()
    html = etree.fromstring(HTMLWRAPPER.format(title=b_json['title']))
    book_elem = etree.SubElement(html, 'body', attrib={'data-type': 'book'})
    title_elem = etree.SubElement(book_elem, 'div',
                                  attrib={'data-type': 'document-title'})
    title_elem.text = b_json['title']
    partcount['book'] += 1

    html_nest(b_json['tree']['contents'], book_elem)

    debug(' '.join(['{}: {}'.format(name, partcount[name]) for name in parts]))
    print(etree.tostring(html), file=html_out)


def html_nest(tree, parent):
    """Recursively construct HTML nested div version of book tree."""
    for node in tree:
        div_elem = etree.SubElement(parent, 'div')
        if node['id'] != 'subcol' and 'contents' not in node:
            page_nodes(node['id'], div_elem)
            set_parent_type(div_elem, parent)

        if args.numchapters is None or partcount['chapter'] < args.numchapters:
            try:
                title_elem = title_xpath(div_elem)[0]
            except IndexError:
                title_elem = etree.Element('div',
                                           attrib={'data-type':
                                                   'document-title'})
                div_elem.insert(0, title_elem)

            title_elem.text = node['title']
            debug(node['title'])

            if 'contents' in node:
                html_nest(node['contents'], div_elem)


def set_parent_type(node, parent):
    if parent is not None and parent.tag != 'body':
        mytype = node.get('data-type')
        mytypeid = parts.index(mytype)
        if parent.get('data-type'):
            parenttype = parent.get('data-type')
            parenttypeid = parts.index(parenttype)
            if parenttypeid <= mytypeid:
                parent.set('data-type', parts[mytypeid + 1])
                partcount[parenttype] -= 1
                partcount[parts[mytypeid + 1]] += 1
        else:
            parent.set('data-type', parts[mytypeid + 1])
            partcount[parts[mytypeid + 1]] += 1

        set_parent_type(parent, parent.getparent())


def page_nodes(page_id, elem):
    """Fetch page return body wrapped in provided element."""

    debug(page_id)
    res = requests.get(ARCHIVEHTML.format(page_id))
    xpath = etree.XPath('//x:body', namespaces=NS)

    body = xpath(etree.fromstring(res.content))[0]

    elem.set('data-type', 'page')
    partcount['page'] += 1
    for c in body.iterchildren():
        elem.append(c)

    return elem


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description="Assemble complete book "
                                                 "as single HTML file")
    parser.add_argument("bookid", help="Identifier of book: "
                        "<uuid|shortId>[@ver]")
    parser.add_argument("html_out", nargs="?",
                        type=argparse.FileType('w'),
                        help="assembled HTML file output (default stdout)",
                        default=sys.stdout)
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Send debugging info to stderr')
    parser.add_argument('-s', '--subset-chapters', dest='numchapters',
                        type=int, const=2, nargs='?', metavar='num_chapters',
                        help="Create subset of complete book "
                        "(default 2 chapters plus extras)")
    args = parser.parse_args()
    verbose = args.verbose
    main(args.bookid, args.html_out)
{'fraction_non_alphanumeric': 0.1483143186770927, 'fraction_numerical': 0.004999359056531214, 'mean_word_length': 2.357142857142857, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5911323', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2304, 'n_words': 489}