<commit_before>import itertools as it
import functools as ft
import multiprocessing as mp
import random
import argparse
import spacy
from .window import Window
from .squirrel import Squirrel
from .methods import Racoon, POSRacoon, MinRacoon, Squirrel
from ..utils import args_to_kwargs, chunks, smart_open, iterlen


LANG = 'de'
BATCH_SIZE = 10000
N_THREADS = 8

methods = {'squirrel': Squirrel,
           'racoon': Racoon,
           'posracoon': POSRacoon,
           'window': Window,
           'minracoon': MinRacoon}


def contexts_for_lines(lines, method):
    # if method.PARSE_TREE:
    #     nlp = spacy.load(LANG)
    # else:
    #     nlp = spacy.load(LANG, parser=False)
    nlp = spacy.load(LANG, parser=False)
    docs = nlp.pipe(lines, batch_size=BATCH_SIZE, n_threads=N_THREADS)
    return (method.lines(doc) for doc in docs)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Create pairs file')
    parser.add_argument('-i', '--input', required=True)
    parser.add_argument('-o', '--output')
    parser.add_argument('-m', '--method',
                        choices=methods.keys(), required=True)
    # parser.add_argument('--max_level', type=int, default=3)
    parser.add_argument('--window', type=int)
    parser.add_argument('-p', '--processes', type=int, default=1)
    parser.add_argument('--fine', action='store_true')

    args = parser.parse_args()
    processes = args.processes
    kwargs = args_to_kwargs(args)
    method = methods[args.method.lower()](**kwargs)

    with open(args.input) as inf, \
            smart_open(args.output) as outf:
        lines = (l.strip() for l in inf)
        lists = contexts_for_lines(lines, method)
        outputlines = (' '.join(context) + '\n' for context in contexts)
        outf.writelines(outputlines)<commit_msg>fix typo<commit_after>import itertools as it
import functools as ft
import multiprocessing as mp
import random
import argparse
import spacy
from .window import Window
from .squirrel import Squirrel
from .methods import Racoon, POSRacoon, MinRacoon, Squirrel
from ..utils import args_to_kwargs, chunks, smart_open, iterlen


LANG = 'de'
BATCH_SIZE = 10000
N_THREADS = 8

methods = {'squirrel': Squirrel,
           'racoon': Racoon,
           'posracoon': POSRacoon,
           'window': Window,
           'minracoon': MinRacoon}


def contexts_for_lines(lines, method):
    # if method.PARSE_TREE:
    #     nlp = spacy.load(LANG)
    # else:
    #     nlp = spacy.load(LANG, parser=False)
    nlp = spacy.load(LANG, parser=False)
    docs = nlp.pipe(lines, batch_size=BATCH_SIZE, n_threads=N_THREADS)
    return (method.lines(doc) for doc in docs)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Create pairs file')
    parser.add_argument('-i', '--input', required=True)
    parser.add_argument('-o', '--output')
    parser.add_argument('-m', '--method',
                        choices=methods.keys(), required=True)
    # parser.add_argument('--max_level', type=int, default=3)
    parser.add_argument('--window', type=int)
    parser.add_argument('-p', '--processes', type=int, default=1)
    parser.add_argument('--fine', action='store_true')

    args = parser.parse_args()
    processes = args.processes
    kwargs = args_to_kwargs(args)
    method = methods[args.method.lower()](**kwargs)

    with open(args.input) as inf, \
            smart_open(args.output) as outf:
        lines = (l.strip() for l in inf)
        contexts = contexts_for_lines(lines, method)
        outputlines = (' '.join(context) + '\n' for context in contexts)
        outf.writelines(outputlines)
{'fraction_non_alphanumeric': 0.1298844248761695, 'fraction_numerical': 0.004402861860209136, 'mean_word_length': 3.296690307328605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6319735', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1188, 'n_words': 280}