<commit_before>"""
Parsers and writers for Web page dump files.
"""
from freequery.document import Document
from cStringIO import StringIO
from disco.comm import Connection as disco_comm_Connection

class WARCFormatError(Exception): pass

class WARCParser(object):

    """Parser for WARC files."""

    def __init__(self, stream):
        self.pos = 0
        if isinstance(stream, str) or isinstance(stream, unicode):
            stream = StringIO(stream)
        elif isinstance(stream, disco_comm_Connection):
            stream = StringIO(stream.read())            
        self.stream = stream

    def tell(self):
        """Returns the cursor position (in bytes) in `iterable`. Used for the
        :class:`Docset` index."""
        return self.pos

    def __parse_header_line(self, line):
        k,v = line.split(":", 1)
        v = v.strip()
        return (k,v)
    
    def next(self):
        # version
        version = self.stream.readline()
        self.pos += len(version)
        if version == '':
            raise StopIteration
        if not version.startswith('WARC/'):
            raise WARCFormatError("record does not start with `version`:" \
                                  " '%s' at pos %d" % (version, self.pos))

        # record header
        warc_type = None
        warc_target_uri = None
        content_length = None
        while True:
            line = self.stream.readline()
            self.pos += len(line)
            if line == "\n":
                break
            else:
                k,v = self.__parse_header_line(line)
                if k == 'WARC-Type':
                    warc_type = v
                elif k == 'WARC-Target-URI':
                    warc_target_uri = v.decode('utf8')
                elif k == 'Content-Length':
                    content_length = int(v)

        # check some required fields
        if warc_type is None:
            raise WARCFormatError("WARC-Type is required")
        if warc_type == 'response' and warc_target_uri is None:
            raise WARCFormatError("WARC-Target-URI is required")
        if content_length is None:
            raise WARCFormatError("Content-Length is required")
                
        # block
        block = self.stream.read(content_length)
        self.pos += content_length        
        
        # remove "\n" from last line of raw, since it delimits
        # the raw from the end-of-document and is not actually part
        # of the raw content
        ##if len(raw) > 0:
        ##    raw[-1] = raw[-1].rstrip()
<commit_msg>ignore unicode errors in WARCParser URI line<commit_after>"""
Parsers and writers for Web page dump files.
"""
from freequery.document import Document
from cStringIO import StringIO
from disco.comm import Connection as disco_comm_Connection

class WARCFormatError(Exception): pass

class WARCParser(object):

    """Parser for WARC files."""

    def __init__(self, stream):
        self.pos = 0
        if isinstance(stream, str) or isinstance(stream, unicode):
            stream = StringIO(stream)
        elif isinstance(stream, disco_comm_Connection):
            stream = StringIO(stream.read())            
        self.stream = stream

    def tell(self):
        """Returns the cursor position (in bytes) in `iterable`. Used for the
        :class:`Docset` index."""
        return self.pos

    def __parse_header_line(self, line):
        k,v = line.split(":", 1)
        v = v.strip()
        return (k,v)
    
    def next(self):
        # version
        version = self.stream.readline()
        self.pos += len(version)
        if version == '':
            raise StopIteration
        if not version.startswith('WARC/'):
            raise WARCFormatError("record does not start with `version`:" \
                                  " '%s' at pos %d" % (version, self.pos))

        # record header
        warc_type = None
        warc_target_uri = None
        content_length = None
        while True:
            line = self.stream.readline()
            self.pos += len(line)
            if line == "\n":
                break
            else:
                k,v = self.__parse_header_line(line)
                if k == 'WARC-Type':
                    warc_type = v
                elif k == 'WARC-Target-URI':
                    # TODO(sqs): handle unicode errors instead of just ignoring
                    # them.
                    warc_target_uri = v.decode('utf8', 'ignore')
                elif k == 'Content-Length':
                    content_length = int(v)

        # check some required fields
        if warc_type is None:
            raise WARCFormatError("WARC-Type is required")
        if warc_type == 'response' and warc_target_uri is None:
            raise WARCFormatError("WARC-Target-URI is required")
        if content_length is None:
            raise WARCFormatError("Content-Length is required")
                
        # block
        block = self.stream.read(content_length)
        self.pos += content_length        
        
        # remove "\n" from last line of raw, since it delimits
        # the raw from the end-of-document and is not actually part
        # of the raw content
        ##if len(raw) > 0:
        ##    raw[-1] = raw[-1].rstrip()

{'fraction_non_alphanumeric': 0.09804665275934003, 'fraction_numerical': 0.0022757443580504457, 'mean_word_length': 1.7383177570093458, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '102006', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1416, 'n_words': 430}