<commit_before>#!/usr/bin/env python

from __future__ import print_function

import argparse
import os
import re
import subprocess
import sys


def run_command(cmd):
    print(' '.join(cmd))
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Utility for testing incremental syntax parsing',
        epilog='''
    Based of a single template the utility generates a pre-edit and a post-edit 
    file. It then verifies that incrementally parsing the post-edit file base 
    on the pre-edit file results in the same syntax tree as reparsing the 
    post-edit file from scratch.

    To generate the pre-edit and the post-edit file from the template, it 
    operates on markers of the form:

        <<test_case<pre|||post>>>

    These placeholders are replaced by:
      - 'pre' if a different test case than 'test_case' is run
      - 'pre' for the pre-edit version of 'test_case'
      - 'post' for the post-edit version of 'test_case'
    ''')
    parser.add_argument(
        'file', type=argparse.FileType(),
        help='The template file to test')
    parser.add_argument(
        '--temp-dir', required=True,
        help='A temporary directory where pre-edit and post-edit files can be \
              saved')<commit_msg>[incrParse] Test utility: Put swift-syntax-test args in quotes if needed<commit_after>#!/usr/bin/env python

from __future__ import print_function

import argparse
import os
import re
import subprocess
import sys


def escapeCmdArg(arg):
    if '"' in arg or ' ' in arg:
        return '"%s"' % arg.replace('"', '\\"')
    else:
        return arg


def run_command(cmd):
    print(' '.join([escapeCmdArg(arg) for arg in cmd]))
    return subprocess.check_output(cmd, stderr=subprocess.STDOUT)


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description='Utility for testing incremental syntax parsing',
        epilog='''
    Based of a single template the utility generates a pre-edit and a post-edit 
    file. It then verifies that incrementally parsing the post-edit file base 
    on the pre-edit file results in the same syntax tree as reparsing the 
    post-edit file from scratch.

    To generate the pre-edit and the post-edit file from the template, it 
    operates on markers of the form:

        <<test_case<pre|||post>>>

    These placeholders are replaced by:
      - 'pre' if a different test case than 'test_case' is run
      - 'pre' for the pre-edit version of 'test_case'
      - 'post' for the post-edit version of 'test_case'
    ''')
    parser.add_argument(
        'file', type=argparse.FileType(),
        help='The template file to test')
    parser.add_argument(
        '--temp-dir', required=True,
        help='A temporary directory where pre-edit and post-edit files can be \
              saved')
{'fraction_non_alphanumeric': 0.08859913072550986, 'fraction_numerical': 0.0, 'mean_word_length': 2.989333333333333, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7293300', 'n_tokens_mistral': 865, 'n_tokens_neox': 830, 'n_words': 318}