<commit_before># Natural Language Toolkit: Gutenberg Corpus Reader
#
# Copyright (C) 2001-2008 NLTK Project
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT

import re, codecs
from plaintext import PlaintextCorpusReader
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.data import FileSystemPathPointer

class GutenbergCorpusReader(PlaintextCorpusReader):
    class CorpusView(StreamBackedCorpusView):
        def __init__(self, filename, block_reader, encoding=None):
            startpos = 0
            
            # Search for a preamble.
            if isinstance(filename, PathPointer):
                stream = filename.open(encoding)
            else:
                stream = FileSystemPathPointer(filename).open(encoding)
            for i in range(300):
                line = stream.readline()
                if line == '':
                    break # No preamble found!
                if re.match(r'\*END\*?\s*THE\s*SMALL\s*PRINT', line):
                    startpos = stream.tell()
                    break # End of the preamble!
            stream.close()
    
            StreamBackedCorpusView.__init__(self, filename,
                                            block_reader, startpos,
                                            encoding=encoding)
    <commit_msg>Added delimiter to Gutenberg texts so that reader can reliably locate the start of each text.<commit_after># Natural Language Toolkit: Gutenberg Corpus Reader
#
# Copyright (C) 2001-2008 NLTK Project
# Author: <NAME> <<EMAIL>>
#         <NAME> <<EMAIL>>
# URL: <http://nltk.org>
# For license information, see LICENSE.TXT

import re, codecs
from plaintext import PlaintextCorpusReader
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.data import FileSystemPathPointer

class GutenbergCorpusReader(PlaintextCorpusReader):
    class CorpusView(StreamBackedCorpusView):
        def __init__(self, filename, block_reader, encoding=None):
            startpos = 0
            
            # Search for a preamble.
            if isinstance(filename, PathPointer):
                stream = filename.open(encoding)
            else:
                stream = FileSystemPathPointer(filename).open(encoding)
            for i in range(400):
                line = stream.readline()
                if line == '':
                    break # No preamble found!
                if line.startswith('@@@@@@@@'):
                    startpos = stream.tell()
                    break # End of the preamble!
            stream.close()
    
            StreamBackedCorpusView.__init__(self, filename,
                                            block_reader, startpos,
                                            encoding=encoding)
    
{'fraction_non_alphanumeric': 0.08730715287517532, 'fraction_numerical': 0.008415147265077139, 'mean_word_length': 1.8731117824773413, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4417016', 'n_tokens_mistral': 778, 'n_tokens_neox': 727, 'n_words': 198}