<commit_before>"""An object to represent warc records, using the abstract record in record.py"""

import re
import hashlib
from .record import ArchiveRecord,ArchiveParser
from .archive_detect import register_record_type

bad_lines = 5 # when to give up looking for the version stamp

@ArchiveRecord.HEADERS(
    DATE='WARC-Date',
    TYPE = 'WARC-Type',
    ID = 'WARC-Record-ID',
    CONCURRENT_TO = 'WARC-Concurrent-To',
    REFERS_TO = 'WARC-Refers-To',
    CONTENT_LENGTH = 'Content-Length',
    CONTENT_TYPE = 'Content-Type',
    URL='WARC-Target-URI',
    BLOCK_DIGEST='WARC-Block-Digest',
    IP_ADDRESS='WARC-IP-Address',
)
class WarcRecord(ArchiveRecord):
    VERSION="WARC/1.0"
    KNOWN_VERSIONS=set(('1.0', '0.16', '0.17')) 
    RESPONSE="response"
    REQUEST="request"
    METADATA="metadata"
    CONVERSION="conversion"

    def __init__(self, version=VERSION, headers=None, content=None, errors=None):
        ArchiveRecord.__init__(self,headers,content,errors) 
        self.version = version

    @property
    def id(self):
        return self.get_header(self.ID)

    def _write_to(self, out, nl):  
        """WARC Format:
            VERSION NL
            (Key: Value NL)* 
            NL
            CONTENT NL
            NL
            
            don't write multi line headers
        """
        out.write(self.version)
        out.write(nl)
        for k,v in self.headers:
            out.write(k)
            out.write(": ")
            out.write(v)
            out.write(nl)
        content_type, content_buffer = self.content
        content_buffer=buffer(content_buffer)
        if content_type:
            out.write(self.CONTENT_TYPE)
            out.write(": ")
            out.write(content_type)
            out.write(nl)
        if content_buffer is None:
            content_buffer=""

        content_length = len(content_buffer)
        out.write(self.CONTENT_LENGTH)
        out.write(": ")
        out.write(str(content_length))
        out.write(nl)

        block_hash = hashlib.sha256()
        block_hash.update(content_buffer)

        digest= "sha256:%s"%block_hash.hexdigest()

        out.write(self.BLOCK_DIGEST)
        out.write(": ")
        out.write(digest)
        out.write(nl)

        # end of header blank nl
        out.write(nl)
        if content_buffer:
            out.write(content_buffer[:content_length])
        out.write(nl)
        out.write(nl)
        out.flush()

    def repair(self):
        pass

    def validate(self):
        return self.errors

    @classmethod
    def make_parser(self):
        return WarcParser()

def rx(pat):
    return re.compile(pat,flags=re.IGNORECASE)

version_rx = rx(r'^(?P<prefix>.*?)(?P<version>\s*WARC/(?P<number>.*?))' '(?P<nl>\r\n|\r|\n)\\Z')
# a header is key: <ws> value plus any following lines with leading whitespace
header_rx = rx(r'^(?P<name>.*?):\s?(?P<value>.*?)' '(?P<nl>\r\n|\r|\n)\\Z')
value_rx = rx(r'^\s+(?P<value>.+?)' '(?P<nl>\r\n|\r|\n)\\Z')
nl_rx=rx('^(?P<nl>\r\n|\r|\n\\Z)')
length_rx = rx('^'+WarcRecord.CONTENT_LENGTH+'$')
type_rx = rx('^'+WarcRecord.CONTENT_TYPE+'$')

required_headers = set((
    WarcRecord.TYPE.lower(), 
    WarcRecord.ID.lower(), 
    WarcRecord.CONTENT_LENGTH.lower(), 
    WarcRecord.DATE.lower(),
))

class WarcParser(ArchiveParser):
    def __init__(self):
        self.trailing_newlines = 0

    def parse(self,stream, offset):
        """Reads a warc record from the stream, returns a tuple (record, errors). 
        Either records is null or errors is null. Any record-specific errors are 
        contained in the record - errors is only used when *nothing* could be parsed"""
        errors = []
        version = None
        # find WARC/.*
        line = stream.readline()
        newlines = self.trailing_newlines
        if newlines > 0:
            while line:
                match = nl_rx.match(line)
                if match and newlines > 0:
                    if offset is not None: offset+=len(line)
                    newlines-=1<commit_msg>whoops<commit_after>"""An object to represent warc records, using the abstract record in record.py"""

import re
import hashlib
from .record import ArchiveRecord,ArchiveParser
from .archive_detect import register_record_type

bad_lines = 5 # when to give up looking for the version stamp

@ArchiveRecord.HEADERS(
    DATE='WARC-Date',
    TYPE = 'WARC-Type',
    ID = 'WARC-Record-ID',
    CONCURRENT_TO = 'WARC-Concurrent-To',
    REFERS_TO = 'WARC-Refers-To',
    CONTENT_LENGTH = 'Content-Length',
    CONTENT_TYPE = 'Content-Type',
    URL='WARC-Target-URI',
    BLOCK_DIGEST='WARC-Block-Digest',
    IP_ADDRESS='WARC-IP-Address',
)
class WarcRecord(ArchiveRecord):
    VERSION="WARC/1.0"
    RESPONSE="response"
    REQUEST="request"
    METADATA="metadata"
    CONVERSION="conversion"

    def __init__(self, version=VERSION, headers=None, content=None, errors=None):
        ArchiveRecord.__init__(self,headers,content,errors) 
        self.version = version

    @property
    def id(self):
        return self.get_header(self.ID)

    def _write_to(self, out, nl):  
        """WARC Format:
            VERSION NL
            (Key: Value NL)* 
            NL
            CONTENT NL
            NL
            
            don't write multi line headers
        """
        out.write(self.version)
        out.write(nl)
        for k,v in self.headers:
            out.write(k)
            out.write(": ")
            out.write(v)
            out.write(nl)
        content_type, content_buffer = self.content
        content_buffer=buffer(content_buffer)
        if content_type:
            out.write(self.CONTENT_TYPE)
            out.write(": ")
            out.write(content_type)
            out.write(nl)
        if content_buffer is None:
            content_buffer=""

        content_length = len(content_buffer)
        out.write(self.CONTENT_LENGTH)
        out.write(": ")
        out.write(str(content_length))
        out.write(nl)

        block_hash = hashlib.sha256()
        block_hash.update(content_buffer)

        digest= "sha256:%s"%block_hash.hexdigest()

        out.write(self.BLOCK_DIGEST)
        out.write(": ")
        out.write(digest)
        out.write(nl)

        # end of header blank nl
        out.write(nl)
        if content_buffer:
            out.write(content_buffer[:content_length])
        out.write(nl)
        out.write(nl)
        out.flush()

    def repair(self):
        pass

    def validate(self):
        return self.errors

    @classmethod
    def make_parser(self):
        return WarcParser()

def rx(pat):
    return re.compile(pat,flags=re.IGNORECASE)

version_rx = rx(r'^(?P<prefix>.*?)(?P<version>\s*WARC/(?P<number>.*?))' '(?P<nl>\r\n|\r|\n)\\Z')
# a header is key: <ws> value plus any following lines with leading whitespace
header_rx = rx(r'^(?P<name>.*?):\s?(?P<value>.*?)' '(?P<nl>\r\n|\r|\n)\\Z')
value_rx = rx(r'^\s+(?P<value>.+?)' '(?P<nl>\r\n|\r|\n)\\Z')
nl_rx=rx('^(?P<nl>\r\n|\r|\n\\Z)')
length_rx = rx('^'+WarcRecord.CONTENT_LENGTH+'$')
type_rx = rx('^'+WarcRecord.CONTENT_TYPE+'$')

required_headers = set((
    WarcRecord.TYPE.lower(), 
    WarcRecord.ID.lower(), 
    WarcRecord.CONTENT_LENGTH.lower(), 
    WarcRecord.DATE.lower(),
))

class WarcParser(ArchiveParser):
    KNOWN_VERSIONS=set(('1.0', '0.16', '0.17')) 
    def __init__(self):
        self.trailing_newlines = 0

    def parse(self,stream, offset):
        """Reads a warc record from the stream, returns a tuple (record, errors). 
        Either records is null or errors is null. Any record-specific errors are 
        contained in the record - errors is only used when *nothing* could be parsed"""
        errors = []
        version = None
        # find WARC/.*
        line = stream.readline()
        newlines = self.trailing_newlines
        if newlines > 0:
            while line:
                match = nl_rx.match(line)
                if match and newlines > 0:
                    if offset is not None: offset+=len(line)
                    newlines-=1
{'fraction_non_alphanumeric': 0.13666293949558952, 'fraction_numerical': 0.005218039508013418, 'mean_word_length': 2.3195876288659796, 'pattern_counts': {'":': 8, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5241600', 'n_tokens_mistral': 2694, 'n_tokens_neox': 2571, 'n_words': 541}