<commit_before>#! /usr/bin/env python3

import os
import sys

from lib.asserts import assert_eq

TEMPLATE_PREFIX = "ntemplate."


def main(args):
    replacement = args[0]

    templatepath = args[1]
    templatefile = os.path.basename(templatepath)
    assert templatefile.startswith(TEMPLATE_PREFIX), templatefile

    outname_template = templatefile[len(TEMPLATE_PREFIX):]
    outname_value = outname_template.replace('N', replacement)

    outpath = args[2]
    outfile = os.path.basename(outpath)

    assert_eq(outname_value, outfile)

    template = open(templatepath, "r").read()
    open(outpath, "w").write(template.format(N=replacement.upper()))
    print(
        "Generated {} from {}".format(os.path.relpath(outpath), templatefile)
    )


if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>utils: n.py: resolve templated includes<commit_after>#! /usr/bin/env python3

import os
import sys
import re

from lib.asserts import assert_eq

TEMPLATE_PREFIX = "ntemplate."


def main(args):
    replacement = args[0]

    templatepath = args[1]
    templatefile = os.path.basename(templatepath)
    assert templatefile.startswith(TEMPLATE_PREFIX), templatefile

    outname_template = templatefile[len(TEMPLATE_PREFIX):]
    outname_value = outname_template.replace('N', replacement)

    outpath = args[2]
    outfile = os.path.basename(outpath)

    assert_eq(outname_value, outfile)

    template = open(templatepath, "r").read()
    template = re.sub(r'(["\s])ntemplate\.N', r'\1{FN}', template)
    open(outpath, "w").write(template.format(N=replacement.upper(), FN=replacement))
    print(
        "Generated {} from {}".format(os.path.relpath(outpath), templatefile)
    )


if __name__ == "__main__":
    main(sys.argv[1:])

{'fraction_non_alphanumeric': 0.13592233009708737, 'fraction_numerical': 0.006282124500285551, 'mean_word_length': 4.2298507462686565, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '556178', 'n_tokens_mistral': 601, 'n_tokens_neox': 593, 'n_words': 109}