<commit_before>"""Various things to do with [x]html that might be useful in more than
one place."""

import lxml, lxml.html, lxml.etree, lxml.html.clean

import os, sys
import re

from urlparse import urlparse, urlsplit, urljoin
from urllib2 import urlopen, HTTPError

OK_TAGS = set([
    "body", "head", "html", "title", "abbr", "acronym", "address",
    "blockquote", "br", "cite", "code", "dfn", "div", "em", "h1", "h2",
    "h3", "h4", "h5", "h6", "kbd", "p", "pre", "q", "samp", "span",
    "strong", "var", "a", "dl", "dt", "dd", "ol", "ul", "li", "object",
    "param", "b", "big", "hr", "i", "small", "sub", "sup", "tt", "del",
    "ins", "bdo", "caption", "col", "colgroup", "table", "tbody", "td",
    "tfoot", "th", "thead", "tr", "img", "area", "map", "meta", "style",
    "link", "base"
    ])

XHTMLNS = '{http://www.w3.org/1999/xhtml}'
XHTML = 'http://www.w3.org/1999/xhtml'

CHAPTER_TEMPLATE = '''<html>
<head>
<title>%(title)s</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
%(text)s
</body>
</html>
'''

#IMG_DIR = '/tmp/images/'
IMG_DIR = '/home/douglas/fm-data/import-tests/staging/'

def log(*messages, **kwargs):
    for m in messages:
        try:
            print >> sys.stderr, m
        except Exception:
            print >> sys.stderr, repr(m)


class Author(object):
    def __init__(self, name, email):
        self.name = name
        self.email = email


class ImageCache(object):
    def __init__(self, cache_dir=IMG_DIR):
        self._fetched = {}
        self.cache_dir = cache_dir

    def read_local_url(self, path):
        f = open(self.cache_dir + path)
        s = f.read()
        f.close()
        return s

    def _save_local_url(self, path, data):
        f = open(self.cache_dir + path, 'w')
        f.write(data)
        f.close()
        #os.chmod(path, 0444)

    #XXX there is something a bit dodgy about this.
    #Two urls could map to the same filename.  (though that is unlikely).
    #
    #XXX use existing images as cache.

    def fetch_if_necessary(self, url, target):
        log(url, target)
        if url in self._fetched:
            if self._fetched[url] == target:
                return target
            log('Not trying to fetch "%s" -> "%s" because stored as "%s"' %(url, target, self._fetched[url]))
            return url
        try:
            f = urlopen(url)
            data = f.read()
            f.close()
        except HTTPError, e:
            #if it is missing, assume it will be missing every time after.
            self._fetched[url] = None
            log(e)
            return url
        self._save_local_url(target, data)
        self._fetched[url] = target
        return target



class BaseChapter(object):
    image_cache = ImageCache()

    def as_html(self):
        """Serialise the tree as html if possible, otherwise wrap the
        original text."""
        if hasattr(self, 'tree'):
            return lxml.etree.tostring(self.tree, method='html')

        return CHAPTER_TEMPLATE % {
            'title': '%s: %s' % (self.book, self.name),
            'text': self.text
        }

    def as_xhtml(self):
        """Convert to xhtml and serialise."""
        try:
            root = self.tree.getroot()
        except AttributeError:
            root = self.tree

        nsmap = {None: XHTML}
        xroot = lxml.etree.Element(XHTMLNS + "html", nsmap=nsmap)

        def descend(el, xel):
            xel.text = el.text
            for k, v in el.items():
                xel.set(k, v)
            for child in el.iterchildren():
                xchild = xel.makeelement(XHTMLNS + child.tag)
                xel.append(xchild)
                descend(child, xchild)
            xel.tail = el.tail

        #xroot.text = root.text
        #xroot.tail = root.tail

        descend(root, xroot)

        return lxml.etree.tostring(xroot)
<commit_msg>Add doctype to xhtml files, which are properly serialised<commit_after>"""Various things to do with [x]html that might be useful in more than
one place."""

import lxml, lxml.html, lxml.etree, lxml.html.clean

import os, sys
import re
from cStringIO import StringIO


from urlparse import urlparse, urlsplit, urljoin
from urllib2 import urlopen, HTTPError

OK_TAGS = set([
    "body", "head", "html", "title", "abbr", "acronym", "address",
    "blockquote", "br", "cite", "code", "dfn", "div", "em", "h1", "h2",
    "h3", "h4", "h5", "h6", "kbd", "p", "pre", "q", "samp", "span",
    "strong", "var", "a", "dl", "dt", "dd", "ol", "ul", "li", "object",
    "param", "b", "big", "hr", "i", "small", "sub", "sup", "tt", "del",
    "ins", "bdo", "caption", "col", "colgroup", "table", "tbody", "td",
    "tfoot", "th", "thead", "tr", "img", "area", "map", "meta", "style",
    "link", "base"
    ])

XHTMLNS = '{http://www.w3.org/1999/xhtml}'
XHTML = 'http://www.w3.org/1999/xhtml'

CHAPTER_TEMPLATE = '''<html>
<head>
<title>%(title)s</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
%(text)s
</body>
</html>
'''

XHTML11_DOCTYPE = '''<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
'''
XML_DEC = '<?xml version="1.0" encoding="UTF-8"?>\n'

IMG_DIR = '/home/douglas/fm-data/import-tests/staging/'

def log(*messages, **kwargs):
    for m in messages:
        try:
            print >> sys.stderr, m
        except Exception:
            print >> sys.stderr, repr(m)


class Author(object):
    def __init__(self, name, email):
        self.name = name
        self.email = email


class ImageCache(object):
    def __init__(self, cache_dir=IMG_DIR):
        self._fetched = {}
        self.cache_dir = cache_dir

    def read_local_url(self, path):
        f = open(self.cache_dir + path)
        s = f.read()
        f.close()
        return s

    def _save_local_url(self, path, data):
        f = open(self.cache_dir + path, 'w')
        f.write(data)
        f.close()
        #os.chmod(path, 0444)

    #XXX there is something a bit dodgy about this.
    #Two urls could map to the same filename.  (though that is unlikely).
    #
    #XXX use existing images as cache.

    def fetch_if_necessary(self, url, target):
        log(url, target)
        if url in self._fetched:
            if self._fetched[url] == target:
                return target
            log('Not trying to fetch "%s" -> "%s" because stored as "%s"' %(url, target, self._fetched[url]))
            return url
        try:
            f = urlopen(url)
            data = f.read()
            f.close()
        except HTTPError, e:
            #if it is missing, assume it will be missing every time after.
            self._fetched[url] = None
            log(e)
            return url
        self._save_local_url(target, data)
        self._fetched[url] = target
        return target



class BaseChapter(object):
    image_cache = ImageCache()

    def as_html(self):
        """Serialise the tree as html if possible, otherwise wrap the
        original text."""
        if hasattr(self, 'tree'):
            return lxml.etree.tostring(self.tree, method='html')

        return CHAPTER_TEMPLATE % {
            'title': '%s: %s' % (self.book, self.name),
            'text': self.text
        }

    def as_xhtml(self):
        """Convert to xhtml and serialise."""
        try:
            root = self.tree.getroot()
        except AttributeError:
            root = self.tree

        nsmap = {None: XHTML}
        xroot = lxml.etree.Element(XHTMLNS + "html", nsmap=nsmap)
        #xtree = xroot.getroottree()

        def xhtml_copy(el, xel):
            xel.text = el.text
            for k, v in el.items():
                xel.set(k, v)
            for child in el.iterchildren():
                xchild = xel.makeelement(XHTMLNS + child.tag)
                xel.append(xchild)
                xhtml_copy(child, xchild)
            xel.tail = el.tail

        xhtml_copy(root, xroot)

        return XML_DEC + XHTML11_DOCTYPE + lxml.etree.tostring(xroot)

{'fraction_non_alphanumeric': 0.1572991793086297, 'fraction_numerical': 0.007336483461825417, 'mean_word_length': 2.2236472945891785, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 1, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5182947', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2696, 'n_words': 734}