<commit_before># coding: utf8
from __future__ import unicode_literals
import argparse
import os
import sys
import traceback

from beastling import __version__
from beastling.beastxml import BeastXml
from beastling.report import BeastlingReport
from beastling.report import BeastlingGeoJSON
import beastling.configuration
from beastling.extractor import extract

wrap_errors = Exception

def errmsg(msg):
    sys.stderr.write(msg)


def main(*args):

    # Parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "config",
        help="Beastling configuration file(s) (or XML file if --extract is used)",
        default=None,
        nargs="+")
    parser.add_argument(
        "--extract",
        default=False,
        action="store_true",
        help="Extract configuration file (and possibly data files) from a BEASTling-generated XML file.")
    parser.add_argument(
        "--report",
        default=False,
        action="store_true",
        help="Save a high-level report on the analysis as a Markdown file.")
    parser.add_argument(
        "--language-list",
        default=False,
        action="store_true",
        help="Save a list of languages in the analysis as a plain text file.")
    parser.add_argument(
        "-o", "--output",
        help="Output filename, for example `-o analysis.xml`",
        default=None)
    parser.add_argument(
        "--overwrite",
        help="Overwrite an existing configuration file.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--stdin",
        help="Read data from stdin.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--prior", "--sample-from-prior", "-p",
        help="Generate XML file which samples from the prior, not posterior.",
        default=False,
        action="store_true")
    parser.add_argument(
        "-v", "--verbose",
        help="Display details of the generated analysis.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--version",
        action="version",
        version = "BEASTling %s" % __version__)
    args = parser.parse_args(args or None)
    if args.extract:
        do_extract(args)
    else:
        do_generate(args)
    sys.exit(0)


def do_extract(args):
    if len(args.config) != 1:
        errmsg("Can only extract from exactly one BEAST XML file")
        sys.exit(1)
    if not os.path.exists(args.config[0]):
        errmsg("No such BEAST XML file: %s\n" % args.config)
        sys.exit(2)
    try:
        messages = extract(args.config[0], args.overwrite)
    except wrap_errors as e:
        errmsg("Error encountered while extracting BEASTling config and/or data files:\n")
        traceback.print_exc()
        sys.exit(3)
    for msg in messages:
        sys.stdout.write(msg)


def do_generate(args):

    # Make sure the requested configuration file exists
    for conf in args.config:
        if not os.path.exists(conf):
            errmsg("No such configuration file: %s\n" % conf)
            sys.exit(1)

    # Build but DON'T PROCESS the Config object
    # This is fast, and gives us enough information to check whether or not
    try:
        config = beastling.configuration.Configuration(
            configfile=args.config, stdin_data=args.stdin, prior=args.prior)
    except wrap_errors as e: # PRAGMA: NO COVER
        errmsg("Error encountered while parsing configuration file:\n")
        traceback.print_exc()
        sys.exit(2)

    # Make sure we can write to the appropriate output filename
    output_filename = args.output if args.output else config.basename+".xml"
    if os.path.exists(output_filename) and not args.overwrite:
        errmsg("File %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % output_filename)
        sys.exit(4)

    # Now that we know we will be able to save the resulting XML, we can take
    # the time to process the config object
    try:
        config.process()
    except wrap_errors as e:
        errmsg("Error encountered while parsing configuration file:\n")
        traceback.print_exc()
        sys.exit(2)
<commit_msg>Small changes for consistent importing.<commit_after># coding: utf8
from __future__ import unicode_literals
import argparse
import os
import sys
import traceback

from beastling import __version__
from beastling.beastxml import BeastXml
from beastling.configuration import Configuration
from beastling.extractor import extract
from beastling.report import BeastlingReport
from beastling.report import BeastlingGeoJSON

wrap_errors = Exception

def errmsg(msg):
    sys.stderr.write(msg)


def main(*args):

    # Parse command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "config",
        help="Beastling configuration file(s) (or XML file if --extract is used)",
        default=None,
        nargs="+")
    parser.add_argument(
        "--extract",
        default=False,
        action="store_true",
        help="Extract configuration file (and possibly data files) from a BEASTling-generated XML file.")
    parser.add_argument(
        "--report",
        default=False,
        action="store_true",
        help="Save a high-level report on the analysis as a Markdown file.")
    parser.add_argument(
        "--language-list",
        default=False,
        action="store_true",
        help="Save a list of languages in the analysis as a plain text file.")
    parser.add_argument(
        "-o", "--output",
        help="Output filename, for example `-o analysis.xml`",
        default=None)
    parser.add_argument(
        "--overwrite",
        help="Overwrite an existing configuration file.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--stdin",
        help="Read data from stdin.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--prior", "--sample-from-prior", "-p",
        help="Generate XML file which samples from the prior, not posterior.",
        default=False,
        action="store_true")
    parser.add_argument(
        "-v", "--verbose",
        help="Display details of the generated analysis.",
        default=False,
        action="store_true")
    parser.add_argument(
        "--version",
        action="version",
        version = "BEASTling %s" % __version__)
    args = parser.parse_args(args or None)
    if args.extract:
        do_extract(args)
    else:
        do_generate(args)
    sys.exit(0)


def do_extract(args):
    if len(args.config) != 1:
        errmsg("Can only extract from exactly one BEAST XML file")
        sys.exit(1)
    if not os.path.exists(args.config[0]):
        errmsg("No such BEAST XML file: %s\n" % args.config)
        sys.exit(2)
    try:
        messages = extract(args.config[0], args.overwrite)
    except wrap_errors as e:
        errmsg("Error encountered while extracting BEASTling config and/or data files:\n")
        traceback.print_exc()
        sys.exit(3)
    for msg in messages:
        sys.stdout.write(msg)


def do_generate(args):

    # Make sure the requested configuration file exists
    for conf in args.config:
        if not os.path.exists(conf):
            errmsg("No such configuration file: %s\n" % conf)
            sys.exit(1)

    # Build but DON'T PROCESS the Config object
    # This is fast, and gives us enough information to check whether or not
    try:
        config = Configuration(
            configfile=args.config, stdin_data=args.stdin, prior=args.prior)
    except wrap_errors as e: # PRAGMA: NO COVER
        errmsg("Error encountered while parsing configuration file:\n")
        traceback.print_exc()
        sys.exit(2)

    # Make sure we can write to the appropriate output filename
    output_filename = args.output if args.output else config.basename+".xml"
    if os.path.exists(output_filename) and not args.overwrite:
        errmsg("File %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % output_filename)
        sys.exit(4)

    # Now that we know we will be able to save the resulting XML, we can take
    # the time to process the config object
    try:
        config.process()
    except wrap_errors as e:
        errmsg("Error encountered while parsing configuration file:\n")
        traceback.print_exc()
        sys.exit(2)

{'fraction_non_alphanumeric': 0.09854489530344257, 'fraction_numerical': 0.0028392286762096296, 'mean_word_length': 2.606655290102389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2307075', 'n_tokens_mistral': 2377, 'n_tokens_neox': 2335, 'n_words': 752}