<commit_before>#!/usr/bin/env python
from __future__ import absolute_import, print_function

import argparse
import codecs
from codecs import StreamWriter  # pylint: disable=unused-import
import collections
import copy
import functools
import io
import logging
import os
import signal
import sys

from typing import (IO, Any, Callable, Dict,  # pylint: disable=unused-import
                    Iterable, List, Mapping, MutableMapping, Optional, Text,
                    TextIO, Tuple, Union, cast)
import pkg_resources  # part of setuptools
import ruamel.yaml as yaml
import schema_salad.validate as validate
from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
from schema_salad.sourceline import strip_dup_lineno
import six
from six import string_types

from . import command_line_tool, workflow
from .argparser import arg_parser, generate_parser, get_default_args
from .cwlrdf import printdot, printrdf
from .errors import UnsupportedRequirement, WorkflowException
from .executors import MultithreadedJobExecutor, SingleJobExecutor
from .load_tool import (  # pylint: disable=unused-import
    FetcherConstructorType, fetch_document, jobloaderctx,
    load_overrides, make_tool, resolve_overrides, resolve_tool_uri,
    validate_document)
from .loghandler import _logger, defaultStreamHandler
from .mutation import MutationManager
from .pack import pack
from .pathmapper import (adjustDirObjs, normalizeFilesDirs, trim_listing,
                         visit_class)
from .process import (Process, scandeps,   # pylint: disable=unused-import
                      shortname, use_custom_schema, use_standard_schema)
from .provenance import ResearchObject
from .resolver import ga4gh_tool_registries, tool_resolver
from .secrets import SecretStore
from .software_requirements import (DependenciesConfiguration,
                                    get_container_from_software_requirements)
from .stdfsaccess import StdFsAccess
from .update import ALLUPDATES, UPDATES
from .utils import (DEFAULT_TMP_PREFIX, add_sizes, json_dumps, onWindows,
                    versionstring, windows_default_container_id,
                    processes_to_kill)
from .context import LoadingContext, RuntimeContext, getdefault
from .builder import HasReqsHints


def _terminate_processes():
    """Kill all spawned processes.

    Processes to be killed must be appended to `utils.processes_to_kill`
    as they are spawned.

    An important caveat: since there's no supported way to kill another
    thread in Python, this function cannot stop other threads from
    continuing to execute while it kills the processes that they've
    spawned. This may occasionally lead to unexpected behaviour.
    """
    # It's possible that another thread will spawn a new task while
    # we're executing, so it's not safe to use a for loop here.
    while processes_to_kill:
        processes_to_kill.popleft().kill()


def _signal_handler(signum, frame):
    """Kill all spawned processes and exit.

    Note that it's possible for another thread to spawn a process after
    all processes have been killed, but before Python exits.

    Refer to the docstring for _terminate_processes() for other caveats.
    """
    _terminate_processes()
    sys.exit(signum)


def generate_example_input(inptype):
    # type: (Union[Text, Dict[Text, Any]]) -> Any
    defaults = {u'null': 'null',
                u'Any': 'null',
                u'boolean': False,
                u'int': 0,
                u'long': 0,
                u'float': 0.1,
                u'double': 0.1,
                u'string': 'default_string',
                u'File': {'class': 'File',
                          'path': 'default/file/path'},
                u'Directory': {'class': 'Directory',
                               'path': 'default/directory/path'}
               }  # type: Dict[Text, Any]
    if (not isinstance(inptype, string_types) and
            not isinstance(inptype, collections.Mapping)
            and isinstance(inptype, collections.MutableSet)):<commit_msg>Add more type hints<commit_after>#!/usr/bin/env python
from __future__ import absolute_import, print_function

import argparse
import codecs
from codecs import StreamWriter  # pylint: disable=unused-import
import collections
import copy
import functools
import io
import logging
import os
import signal
import sys

from typing import (IO, Any, Callable, Dict,  # pylint: disable=unused-import
                    Iterable, List, Mapping, MutableMapping, Optional, Text,
                    TextIO, Tuple, Union, cast)
import pkg_resources  # part of setuptools
import ruamel.yaml as yaml
import schema_salad.validate as validate
from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
from schema_salad.sourceline import strip_dup_lineno
import six
from six import string_types

from . import command_line_tool, workflow
from .argparser import arg_parser, generate_parser, get_default_args
from .cwlrdf import printdot, printrdf
from .errors import UnsupportedRequirement, WorkflowException
from .executors import MultithreadedJobExecutor, SingleJobExecutor
from .load_tool import (  # pylint: disable=unused-import
    FetcherConstructorType, fetch_document, jobloaderctx,
    load_overrides, make_tool, resolve_overrides, resolve_tool_uri,
    validate_document)
from .loghandler import _logger, defaultStreamHandler
from .mutation import MutationManager
from .pack import pack
from .pathmapper import (adjustDirObjs, normalizeFilesDirs, trim_listing,
                         visit_class)
from .process import (Process, scandeps,   # pylint: disable=unused-import
                      shortname, use_custom_schema, use_standard_schema)
from .provenance import ResearchObject
from .resolver import ga4gh_tool_registries, tool_resolver
from .secrets import SecretStore
from .software_requirements import (DependenciesConfiguration,
                                    get_container_from_software_requirements)
from .stdfsaccess import StdFsAccess
from .update import ALLUPDATES, UPDATES
from .utils import (DEFAULT_TMP_PREFIX, add_sizes, json_dumps, onWindows,
                    versionstring, windows_default_container_id,
                    processes_to_kill)
from .context import LoadingContext, RuntimeContext, getdefault
from .builder import HasReqsHints


def _terminate_processes():
    # type: () -> None
    """Kill all spawned processes.

    Processes to be killed must be appended to `utils.processes_to_kill`
    as they are spawned.

    An important caveat: since there's no supported way to kill another
    thread in Python, this function cannot stop other threads from
    continuing to execute while it kills the processes that they've
    spawned. This may occasionally lead to unexpected behaviour.
    """
    # It's possible that another thread will spawn a new task while
    # we're executing, so it's not safe to use a for loop here.
    while processes_to_kill:
        processes_to_kill.popleft().kill()


def _signal_handler(signum, frame):
    # type: (int, Any) -> None
    """Kill all spawned processes and exit.

    Note that it's possible for another thread to spawn a process after
    all processes have been killed, but before Python exits.

    Refer to the docstring for _terminate_processes() for other caveats.
    """
    _terminate_processes()
    sys.exit(signum)


def generate_example_input(inptype):
    # type: (Union[Text, Dict[Text, Any]]) -> Any
    defaults = {u'null': 'null',
                u'Any': 'null',
                u'boolean': False,
                u'int': 0,
                u'long': 0,
                u'float': 0.1,
                u'double': 0.1,
                u'string': 'default_string',
                u'File': {'class': 'File',
                          'path': 'default/file/path'},
                u'Directory': {'class': 'Directory',
                               'path': 'default/directory/path'}
               }  # type: Dict[Text, Any]
    if (not isinstance(inptype, string_types) and
            not isinstance(inptype, collections.Mapping)
            and isinstance(inptype, collections.MutableSet)):
{'fraction_non_alphanumeric': 0.0700269806230071, 'fraction_numerical': 0.0017169487368162864, 'mean_word_length': 3.1799077396207074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '296879', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2203, 'n_words': 748}