<commit_before>#!/usr/bin/env python3

import click

from queryexpander.expansion import QueryExpander


@click.group()
def cli():
    pass


@click.command()
@click.argument('query', required=True, type=str)
@click.argument('vocabulary_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('number_of_additional_words', required=True, type=int)
def expand(query: str, vocabulary_path: str, number_of_additional_words: int):
    query_expander = QueryExpander(vocabulary_path)
    print(query_expander.find_most_similar_words(query.rstrip('\n').lower().split(' '), number_of_additional_words))


@click.command()
@click.argument('input_file_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('output_file_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('vocabulary_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('number_of_additional_words', required=True, type=int)
def expand_file(input_file_path: str, output_file_path: str, vocabulary_path: str, number_of_additional_words: int):
    query_expander = QueryExpander(vocabulary_path)

    with open(input_file_path) as input_file:
        with open(output_file_path, 'w') as output_file:
            for line in input_file:
                output_file.write(query_expander.find_most_similar_words(line.rstrip('\n').lower().split(' '),
                                                                         number_of_additional_words))

cli.add_command(expand)
cli.add_command(expand_file)

if __name__ == '__main__':
    cli()<commit_msg>Add generating cache from the CLI<commit_after>#!/usr/bin/env python3

import click

from queryexpander.expansion import QueryExpander


@click.group()
def cli():
    pass


@click.command()
@click.argument('vocabulary_path', required=True, type=click.Path(exists=True, dir_okay=False))
def generate_cache(vocabulary_path: str):
    query_expander = QueryExpander(vocabulary_path)
    query_expander.generate_sums_cache()


@click.command()
@click.argument('query', required=True, type=str)
@click.argument('vocabulary_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('number_of_additional_words', required=True, type=int)
def expand(query: str, vocabulary_path: str, number_of_additional_words: int):
    query_expander = QueryExpander(vocabulary_path)
    print(query_expander.find_most_similar_words(query.rstrip('\n').lower().split(' '), number_of_additional_words))


@click.command()
@click.argument('input_file_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('output_file_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('vocabulary_path', required=True, type=click.Path(exists=True, dir_okay=False))
@click.argument('number_of_additional_words', required=True, type=int)
def expand_file(input_file_path: str, output_file_path: str, vocabulary_path: str, number_of_additional_words: int):
    query_expander = QueryExpander(vocabulary_path)

    with open(input_file_path) as input_file:
        with open(output_file_path, 'w') as output_file:
            for line in input_file:
                output_file.write(query_expander.find_most_similar_words(line.rstrip('\n').lower().split(' '),
                                                                         number_of_additional_words))

cli.add_command(generate_cache)
cli.add_command(expand)
cli.add_command(expand_file)

if __name__ == '__main__':
    cli()
{'fraction_non_alphanumeric': 0.12507026419336706, 'fraction_numerical': 0.0005621135469364812, 'mean_word_length': 5.401079136690647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5292758', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1131, 'n_words': 180}