<commit_before>"""
This code includes functions to preprocess text in bookcorpus dataset.
11,038 books are included.
See "Skip-Thought Vectors" for bookcorpus usage.
(<NAME>, <NAME> et al., 2015.)
Contact authors to get bookcorpus dataset.

Using two text files.
Base datapath: '~~~/data/'
Additional foder structure: '~~~/data/bookcorpus/books_large_p1.txt' (p2.txt)

First, preprocess bookcorpus to make pickle.
Second, load it to class.
"""

import time
import pickle
import numpy as np
from lemontree.data.dataset import BaseDataset
from lemontree.data.glove import GloveData
from lemontree.utils.data_utils import split_data


def read_bookcorpus(base_datapath, glove, mode='books_large_p1',
                    num_sentence=100000, min_length=2, max_length=100, seed=9685):
    """
    This function read bookcorpus data, tokenize to word-level, select and save sentences.
    For default, only put '<EOS>' at the end of the sentence.
    Save pickle of selected sentences for reuse.

    Parameters
    ----------
    base_datapath: string
        a string path where bookcorpus is saved.
    glove: GloveData
        a GloveData which includes the embeddings and dictionary.
    mode: string, default: 'books_large_p1'
        a string which file will be read. {'books_large_p1', 'books_large_p2'}
    num_sentences: integer, default: 100000
        an integer that how much sentences will be selected.
        the number is not yet filtered by length.
    min_length: integer, default: 1
        an integer which indicates minimum length of the sentence.
    max_length: integer, default: 100
        an integer which indicates maximum length of the sentence.
    seed: integer, default: 9685
        a random seed to select random sentences
    
    Returns
    -------
    None.
    """
    # check asserts
    assert isinstance(base_datapath, str), '"base_datapath" should be a string path.'
    assert isinstance(glove, GloveData), '"glove" should be GloveData instance.'
    assert mode in ['books_large_p1', 'books_large_p2'], '"mode" should be one name of the two files.'
    assert isinstance(num_sentence, int), '"num_sentence" should be a positive integer.'
    assert isinstance(min_length, int), '"min_length" should be a positive integer.'
    assert isinstance(max_length, int), '"max_length" should be a positive integer.'
    assert min_length < max_length, '"max_length" should be larger than "min_length".'
    assert isinstance(seed, int), '"seed" should be a positive integer.'

    file_name = base_datapath + 'bookcorpus/' + mode + '.txt'
    print('BookCorpus load corpus:', file_name)
    # glove = GloveData(base_datapath, load_pickle = True)
    
    import nltk
    # nltk.download()
    # download punkt module if not exist

    with open(file_name, 'r') as f:
        start_time = time.clock()
        corpus = f.read().lower().replace('\n', ' ')  # remove linebreak
        end_time = time.clock()
        print('BookCorpus load time:', end_time - start_time)

        # sentence tokenize all
        start_time = time.clock()
        sentences = nltk.tokenize.sent_tokenize(corpus)  # list of string
        end_time = time.clock()
        print('NLTK tokenize time:', end_time - start_time)<commit_msg>bookcorpus change<commit_after>"""
This code includes functions to preprocess text in bookcorpus dataset.
11,038 books are included.
See "Skip-Thought Vectors" for bookcorpus usage.
(<NAME>, <NAME> et al., 2015.)
Contact authors to get bookcorpus dataset.

Using two text files.
Base datapath: '~~~/data/'
Additional foder structure: '~~~/data/bookcorpus/books_large_p1.txt' (p2.txt)

First, preprocess bookcorpus to make pickle.
Second, load it to class.
"""

import time
import pickle
import numpy as np
from lemontree.data.dataset import BaseDataset
from lemontree.data.glove import GloveData
from lemontree.utils.data_utils import split_data


def read_bookcorpus(base_datapath, glove, mode='books_large_p1',
                    num_sentence=100000, min_length=2, max_length=100, seed=9685):
    """
    This function read bookcorpus data, tokenize to word-level, select and save sentences.
    For default, only put '<EOS>' at the end of the sentence.
    Save pickle of selected sentences for reuse.

    Parameters
    ----------
    base_datapath: string
        a string path where bookcorpus is saved.
    glove: GloveData
        a GloveData which includes the embeddings and dictionary.
    mode: string, default: 'books_large_p1'
        a string which file will be read. {'books_large_p1', 'books_large_p2'}
    num_sentences: integer, default: 100000
        an integer that how much sentences will be selected.
        the number is not yet filtered by length.
    min_length: integer, default: 1
        an integer which indicates minimum length of the sentence.
    max_length: integer, default: 100
        an integer which indicates maximum length of the sentence.
    seed: integer, default: 9685
        a random seed to select random sentences
    
    Returns
    -------
    None.
    """
    # check asserts
    assert isinstance(base_datapath, str), '"base_datapath" should be a string path.'
    assert mode in ['books_large_p1', 'books_large_p2'], '"mode" should be one name of the two files.'
    assert isinstance(num_sentence, int), '"num_sentence" should be a positive integer.'
    assert isinstance(min_length, int), '"min_length" should be a positive integer.'
    assert isinstance(max_length, int), '"max_length" should be a positive integer.'
    assert min_length < max_length, '"max_length" should be larger than "min_length".'
    assert isinstance(seed, int), '"seed" should be a positive integer.'

    file_name = base_datapath + 'bookcorpus/' + mode + '.txt'
    print('BookCorpus load corpus:', file_name)
    # glove = GloveData(base_datapath, load_pickle = True)
    
    import nltk
    # nltk.download()
    # download punkt module if not exist

    with open(file_name, 'r') as f:
        start_time = time.clock()
        corpus = f.read().lower().replace('\n', ' ')  # remove linebreak
        end_time = time.clock()
        print('BookCorpus load time:', end_time - start_time)

        # sentence tokenize all
        start_time = time.clock()
        sentences = nltk.tokenize.sent_tokenize(corpus)  # list of string
        end_time = time.clock()
        print('NLTK tokenize time:', end_time - start_time)
{'fraction_non_alphanumeric': 0.09439850210641286, 'fraction_numerical': 0.014042752379466375, 'mean_word_length': 3.2876254180602005, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2617741', 'n_tokens_mistral': 1953, 'n_tokens_neox': 1859, 'n_words': 674}