<commit_before>import numpy as np
import itertools
import nltk

UNKNOWN_TOKEN = 'UNKNOWN_TOKEN'

data = {}
current_epoch = 0
position_in_batch = 0


def read(filename, vocab_fraction=0, sentence_tokenized=False, is_string=False):
    """
    Read raw data and add data properties to self.data

    Args:
        filename: String. Name of file to read raw data from
        tokenize: Use NLTK to tokenize (word or sentence), or split into chars
        tokenize_type: if tokenizing, whether to use word or sentence tokenization

    Returns:
        dict: python dictionary containing data properties
    """
    # TODO: change name of variable 'filename' to reflect that it can also be a string of the raw data
    # TODO: add paramter 'is_string' to docstring
    global data

    # load raw data
    if is_string:
        raw_data = filename
    else:
        raw_data = open(filename, 'r').read()

    # sentence tokenize data
    if not sentence_tokenized:
        sentences = nltk.sent_tokenize(raw_data)<commit_msg>added documentation for keys in data<commit_after>import numpy as np
import itertools
import nltk

UNKNOWN_TOKEN = 'UNKNOWN_TOKEN'

data = {}
"""
data can contain the following keys:
sentences: list of sentence strings
tokens: word-tokenized list of sentences
unique_tokens: list of unique tokens sorted by frequency
indices: integer representation of tokens
one_hot: one-hot representation of indices
vocab_size: number of unique tokens
token_to_index: map from token to index space
index_to_token: map from index to token space
max_sentence_length: number of tokens in the longest sentence
raw_data: full text
data_size: number of tokens in data
filename: name of file to read raw data from
tokens_flat: flattened sequence of word tokens
token_to_one_hot: map from tokens to one-hot space
index_to_one_hot: map from indices to one-hot space
"""
current_epoch = 0
position_in_batch = 0


def read(filename, vocab_fraction=0, sentence_tokenized=False, is_string=False):
    """
    Read raw data and add data properties to self.data

    Args:
        filename: String. Name of file to read raw data from
        tokenize: Use NLTK to tokenize (word or sentence), or split into chars
        tokenize_type: if tokenizing, whether to use word or sentence tokenization

    Returns:
        dict: python dictionary containing data properties
    """
    # TODO: change name of variable 'filename' to reflect that it can also be a string of the raw data
    # TODO: add paramter 'is_string' to docstring
    global data

    # load raw data
    if is_string:
        raw_data = filename
    else:
        raw_data = open(filename, 'r').read()

    # sentence tokenize data
    if not sentence_tokenized:
        sentences = nltk.sent_tokenize(raw_data)
{'fraction_non_alphanumeric': 0.05631768953068592, 'fraction_numerical': 0.0021660649819494585, 'mean_word_length': 3.5650741350906094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2305374', 'n_tokens_mistral': 780, 'n_tokens_neox': 771, 'n_words': 333}