<jupyter_start><jupyter_text>Benchmarking different tokening approaches  fastai v1.0<jupyter_code>from fastai.text import *
path = untar_data(URLs.IMDB)
il = (TextList.from_folder(path, processor=[OpenFileProcessor(), TokenizeProcessor()])
              .filter_by_folder(include=['train', 'test', 'unsup']))
opener = OpenFileProcessor()
opener.process(il)
tokenizer = TokenizeProcessor()
%time tokenizer.process(il)<jupyter_output><empty_output><jupyter_text>Memory peak at 3.42G (389MB without the kernel)  dev_course nb 12<jupyter_code>from exp.nb_12 import *
path = datasets.untar_data(datasets.URLs.IMDB)
il = TextList.from_files(path, include=['train', 'test', 'unsup'])
tp = TokenizeProcessor()
texts = [read_file(f) for f in il.items]
%time tokens = tp(texts)<jupyter_output><empty_output><jupyter_text>Peak at 4.6GDoesn't kill process each time  Other<jupyter_code>from exp.nb_12 import *
path = datasets.untar_data(datasets.URLs.IMDB)
il = TextList.from_files(path, include=['train', 'test', 'unsup'])
from multiprocessing import Process, Queue, cpu_count
def text_gen(fnames):
    for fn in fnames:
        with open(fn, 'r') as r:
            txt = r.read()
            for fn in default_pre_rules:
                txt = fn(txt)
            yield txt
def process_files(fnames, data_queue, progress_queue, lang='en', batch_size=5000):
    nlp = spacy.blank(lang, disable=["parser", "tagger", "ner"])
    for w in default_spec_tok: nlp.tokenizer.add_special_case(w, [{ORTH: w}])
    tokens = []
    for docs in nlp.pipe(text_gen(fnames), batch_size=batch_size):
        toks = [t.text for t in docs]
        for fn in default_post_rules: toks = fn(toks)
        tokens.append(toks)
        progress_queue.put(1)
    data_queue.put(tokens)
def tokenize(fnames, lang='en', n_workers=4, chunk_size=5000):
    progress_queue,data_queue = Queue(maxsize=n_workers),Queue(maxsize=n_workers)
    processes = [Process(target=process_files,
                         args=(batch, data_queue, progress_queue, lang, chunk_size))
                 for i,batch in enumerate(np.array_split(fnames, n_workers))]
    for p in processes: p.start()
    tokens = []
    for _ in progress_bar(fnames): _ = progress_queue.get()  
    for _ in processes: tokens += data_queue.get()
    for p in processes: p.join()
    return tokens
%time t = tokenize(il.items)<jupyter_output><empty_output><jupyter_text>Writing tokens in memory<jupyter_code>from exp.nb_12 import *
path = datasets.untar_data(datasets.URLs.IMDB)
il = TextList.from_files(path, include=['train', 'test', 'unsup'])
from multiprocessing import Process, Queue, cpu_count
def text_gen(fnames):
    for fn in fnames:
        with open(fn, 'r') as r:
            txt = r.read()
            for fn in default_pre_rules:
                txt = fn(txt)
            yield txt
def process_files(fnames, output_file, data_queue, progress_queue, lang='en', batch_size=5000):
    nlp = spacy.blank(lang, disable=["parser", "tagger", "ner"])
    for w in default_spec_tok: nlp.tokenizer.add_special_case(w, [{ORTH: w}])
    counts = Counter()
    with open(output_file, 'w') as w:
        for docs in nlp.pipe(text_gen(fnames), batch_size=batch_size):
            tokens = [t.text for t in docs]
            for fn in default_post_rules: tokens = fn(tokens)
            w.write(' '.join(tokens) + ' ')
            progress_queue.put(1)
            counts.update(Counter(tokens))
    data_queue.put(counts)
def tokenize(fnames, lang='en', n_workers=4, chunk_size=5000, tokens_dir='tmp'):
    progress_queue,data_queue = Queue(maxsize=n_workers),Queue(maxsize=n_workers)
    tokens_dir = Path(tokens_dir)
    os.makedirs(tokens_dir, exist_ok=True)
    processes = [Process(target=process_files,
                         args=(batch, tokens_dir/f'tokens{i}.txt', data_queue, progress_queue, lang, chunk_size))
                 for i,batch in enumerate(np.array_split(fnames, n_workers))]
    
    for p in processes: p.start()
    counter = Counter()
    for _ in progress_bar(range(len(fnames))): _ = progress_queue.get()  
    for _ in processes: counter.update(data_queue.get())
    for p in processes: p.join()
    return counter
%time counter = tokenize(il.items)
itos = [v for (v,c) in counter.most_common()[:60000] if c >= 2]
for o in reversed(default_spec_tok):
    if o in itos: itos.remove(o)
    itos.insert(0, o)
stoi = collections.defaultdict(int,{s:i for i,s in enumerate(itos)})<jupyter_output><empty_output><jupyter_text>Tokens are all written in a rwo (there are \n in some texts in IMDB that might make lines, but there might not always be) and we don't want to read all in one go since the goal is to spare memory, so we read by chunks.<jupyter_code>def read_chunks(file, chunk_size=1024):
    while True:
        data = file.read(chunk_size)
        if not data: break
        yield data<jupyter_output><empty_output><jupyter_text>Token EOS tells us when we have finished a sentence.<jupyter_code>def process_tokens(fname, stoi, data_queue, pid):
    all_ids,ids,last = [],[],''
    with open(fname, 'r') as f:
        for chunk in read_chunks(f):
            stream = (last+chunk).split(' ')
            for t in stream[:-1]: #Last token is incomplete (probably) so we keep it for the next chunk
                ids.append(stoi[t])
                if t == EOS:
                    all_ids.append(ids)
                    ids = []
            last = stream[-1]
    data_queue.put([pid, all_ids])
def numericalize(stoi, tokens_dir='tmp', n_workers=4):
    data_queue = Queue(maxsize=n_workers)
    tokens_dir = Path(tokens_dir)
    processes = [Process(target=process_tokens,
                         args=(tokens_dir/f'tokens{i}.txt', stoi, data_queue, i))
                 for i in range(n_workers)]
    for p in processes: p.start()
    ids = [data_queue.get() for _ in processes]
    for p in processes: p.join()
    ids.sort(key = lambda x:x[0])
    return np.concatenate([o[1] for o in ids])
%time ids = numericalize(stoi)
len(ids)
' '.join([itos[i] for i in ids[-1]])
with open(il.items[-1], 'r') as f: print(f.read())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12526608809562798, 'fraction_numerical': 0.00916980514164074, 'mean_word_length': 3.501105379513633, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '365826', 'license': 'Apache-2.0', 'n_tokens_mistral': 2107, 'n_tokens_neox': 2057, 'n_words': 510, 'path': 'dev_nb/202_tokenizing.ipynb', 'repo_name': 'raghavab1992/fav3part2', 'size': '6107'}