<commit_before><commit_msg>Add script to convert frog output to saf<commit_after>#!/usr/bin/env python
import click
import os
import codecs
import json

from xtas.tasks._frog import parse_frog, frog_to_saf


@click.command()
@click.argument('input_dir', type=click.Path(exists=True))
@click.argument('output_dir', type=click.Path())
def frog2saf(input_dir, output_dir):
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    frog_files = os.listdir(input_dir)
    for fi in frog_files:
        with codecs.open(os.path.join(input_dir, fi)) as f:
            lines = f.readlines()
            lines = [line.strip() for line in lines]
        saf_data = frog_to_saf(parse_frog(lines))

        out_file = os.path.join(output_dir, '{}.json'.format(fi))
        with codecs.open(out_file, 'wb', encoding='utf-8') as f:
            json.dump(saf_data, f, indent=4)


if __name__ == '__main__':
    frog2saf()
{'fraction_non_alphanumeric': 0.13095238095238096, 'fraction_numerical': 0.004329004329004329, 'mean_word_length': 3.5792079207920793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '623520', 'n_tokens_mistral': 337, 'n_tokens_neox': 327, 'n_words': 71}