<commit_before>"""
Helper script for starting up bolts and spouts.
"""

import argparse
import importlib
import os
import sys

from pystorm.component import _SERIALIZERS


def main():
    """main entry point for Python bolts and spouts"""
    parser = argparse.ArgumentParser(description='Run a bolt/spout class',
                                     epilog='This is internal to streamparse '
                                            'and is used to run spout and bolt '
                                            'classes via ``python -m '
                                            'streamparse.run <class name>``.')
    parser.add_argument('target_class', help='The bolt/spout class to start.')
    parser.add_argument('-s', '--serializer',
                        help='The serialization protocol to use to talk to '
                             'Storm.',
                        choices=_SERIALIZERS.keys(),
                        default='json')
    args = parser.parse_args()
    # Add current directory to sys.path so imports will work
    sys.path.append(os.getcwd())
    # Import module
    mod_name, cls_name = args.target_class.rsplit('.', 1)
    mod = importlib.import_module(mod_name)
    # Get class from module and run it
    cls = getattr(mod, cls_name)
    cls(serializer=args.serializer).run()


if __name__ == '__main__':
    main()<commit_msg>Work around Storm sending all arguments to streamparse_run as a single string<commit_after>"""
Helper script for starting up bolts and spouts.
"""

import argparse
import importlib
import os
import sys

from pystorm.component import _SERIALIZERS


def main():
    """main entry point for Python bolts and spouts"""
    parser = argparse.ArgumentParser(description='Run a bolt/spout class',
                                     epilog='This is internal to streamparse '
                                            'and is used to run spout and bolt '
                                            'classes via ``python -m '
                                            'streamparse.run <class name>``.')
    parser.add_argument('target_class', help='The bolt/spout class to start.')
    parser.add_argument('-s', '--serializer',
                        help='The serialization protocol to use to talk to '
                             'Storm.',
                        choices=_SERIALIZERS.keys(),
                        default='json')
    # Storm sends everything as one string, which is not great
    if len(sys.argv) == 2:
        sys.argv = [sys.argv[0]] + sys.argv[1].split()
    args = parser.parse_args()
    # Add current directory to sys.path so imports will work
    sys.path.append(os.getcwd())
    # Import module
    mod_name, cls_name = args.target_class.rsplit('.', 1)
    mod = importlib.import_module(mod_name)
    # Get class from module and run it
    cls = getattr(mod, cls_name)
    cls(serializer=args.serializer).run()


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.09698202780603594, 'fraction_numerical': 0.00169548999660902, 'mean_word_length': 1.9006882989183873, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3697838', 'n_tokens_mistral': 783, 'n_tokens_neox': 764, 'n_words': 246}