<commit_before>#!/usr/bin/env python3
#
# Generate a schema for mosaic
import argparse
import subprocess
import re
import os

def do_schema(k3_file, out_file=None):
    sand_path = subprocess.check_output('find .cabal-sandbox -name "*packages.conf.d"',shell=True)
    sand_arg = '--mpargs package-db=' + sand_path.decode("utf-8")
    sand_arg = sand_arg[:-1] # remove newline
    args = ['./dist/build/k3/k3', '-I ./lib/k3', '-I ./examples/sql', 'compile',
        '-l ktrace',
        '--fstage none=True',
        sand_arg,
        '--mpsearch ./src',
        k3_file,
        '2>&1']
    args2 = " ".join(args)
    output = subprocess.check_output(args2, shell=True, stderr=subprocess.STDOUT)
    output = output.decode("utf-8")
    # search for 'drop table' as a start signal
    mo = re.search('(drop table.*)', output, flags=re.DOTALL)
    # from this point to the end
    if mo:
        schema = mo.group(1)
        if out_file:
            with open(out_file, 'w') as f:
                f.write(schema)
        else:
            print(schema)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("k3_file", type=str, help="Specify path of k3 file")
    args = parser.parse_args()
    run_test(args.k3_file)

if __name__=='__main__':
  main ()<commit_msg>mosaic: minor fix for create_schema<commit_after>#!/usr/bin/env python3
#
# Generate a schema for mosaic
import argparse
import subprocess
import re
import os

def do_schema(k3_file, out_file=None):
    sand_path = subprocess.check_output('find .cabal-sandbox -name "*packages.conf.d"',shell=True)
    sand_arg = '--mpargs package-db=' + sand_path.decode("utf-8")
    sand_arg = sand_arg[:-1] # remove newline
    args = ['./dist/build/k3/k3', '-I ./lib/k3', '-I ./examples/sql', 'compile',
        '-l ktrace',
        '--fstage none=True',
        sand_arg,
        '--mpsearch ./src',
        k3_file,
        '2>&1']
    args2 = " ".join(args)
    output = subprocess.check_output(args2, shell=True, stderr=subprocess.STDOUT)
    output = output.decode("utf-8")
    # search for 'drop table' as a start signal
    mo = re.search('(drop table.*)', output, flags=re.DOTALL)
    # from this point to the end
    if mo:
        schema = mo.group(1)
        if out_file:
            with open(out_file, 'w') as f:
                f.write(schema)
        else:
            print(schema)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("k3_file", type=str, help="Specify path of k3 file")
    args = parser.parse_args()
    do_schema(args.k3_file)

if __name__=='__main__':
  main ()
{'fraction_non_alphanumeric': 0.14847666795217895, 'fraction_numerical': 0.013112225221750868, 'mean_word_length': 2.689900426742532, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000552', 'n_tokens_mistral': 916, 'n_tokens_neox': 904, 'n_words': 214}