<commit_before># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This module defines a generic Launcher for all TFleX nodes."""

import sys
import traceback
from typing import Any, Dict, List, Mapping, Optional, Type, TypeVar

from absl import logging
import attr
import grpc
import portpicker
from tfx import types
from tfx.dsl.io import fileio
from tfx.orchestration import data_types_utils
from tfx.orchestration import metadata
from tfx.orchestration.portable import base_driver_operator
from tfx.orchestration.portable import base_executor_operator
from tfx.orchestration.portable import beam_executor_operator
from tfx.orchestration.portable import cache_utils
from tfx.orchestration.portable import data_types
from tfx.orchestration.portable import docker_executor_operator
from tfx.orchestration.portable import execution_publish_utils
from tfx.orchestration.portable import execution_watcher
from tfx.orchestration.portable import importer_node_handler
from tfx.orchestration.portable import inputs_utils
from tfx.orchestration.portable import outputs_utils
from tfx.orchestration.portable import python_driver_operator
from tfx.orchestration.portable import python_executor_operator
from tfx.orchestration.portable import resolver_node_handler
from tfx.orchestration.portable.input_resolution import exceptions
from tfx.orchestration.portable.mlmd import context_lib
from tfx.orchestration.portable.mlmd import execution_lib
from tfx.proto.orchestration import driver_output_pb2
from tfx.proto.orchestration import executable_spec_pb2
from tfx.proto.orchestration import execution_result_pb2
from tfx.proto.orchestration import pipeline_pb2
from tfx.utils import typing_utils

from google.protobuf import message
from ml_metadata.proto import metadata_store_pb2

# Subclasses of BaseExecutorOperator
ExecutorOperator = TypeVar(
    'ExecutorOperator', bound=base_executor_operator.BaseExecutorOperator)

# Subclasses of BaseDriverOperator
DriverOperator = TypeVar(
    'DriverOperator', bound=base_driver_operator.BaseDriverOperator)

DEFAULT_EXECUTOR_OPERATORS = {
    executable_spec_pb2.PythonClassExecutableSpec:
        python_executor_operator.PythonExecutorOperator,
    executable_spec_pb2.BeamExecutableSpec:
        beam_executor_operator.BeamExecutorOperator,
    executable_spec_pb2.ContainerExecutableSpec:
        docker_executor_operator.DockerExecutorOperator
}

DEFAULT_DRIVER_OPERATORS = {
    executable_spec_pb2.PythonClassExecutableSpec:
        python_driver_operator.PythonDriverOperator
}

# LINT.IfChange
_SYSTEM_NODE_HANDLERS = {
    'tfx.dsl.components.common.importer.Importer':
        importer_node_handler.ImporterNodeHandler,
    'tfx.dsl.components.common.resolver.Resolver':
        resolver_node_handler.ResolverNodeHandler,
}
# LINT.ThenChange(Internal system node list)

_ERROR_CODE_UNIMPLEMENTED: int = grpc.StatusCode.UNIMPLEMENTED.value[0]
<commit_msg>no op.<commit_after># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This module defines a generic Launcher for all TFleX nodes."""

import sys
import traceback
from typing import Any, Dict, List, Mapping, Optional, Type, TypeVar

from absl import logging
import attr
import grpc
import portpicker
from tfx import types
from tfx.dsl.io import fileio
from tfx.orchestration import data_types_utils
from tfx.orchestration import metadata
from tfx.orchestration.portable import base_driver_operator
from tfx.orchestration.portable import base_executor_operator
from tfx.orchestration.portable import beam_executor_operator
from tfx.orchestration.portable import cache_utils
from tfx.orchestration.portable import data_types
from tfx.orchestration.portable import docker_executor_operator
from tfx.orchestration.portable import execution_publish_utils
from tfx.orchestration.portable import execution_watcher
from tfx.orchestration.portable import importer_node_handler
from tfx.orchestration.portable import inputs_utils
from tfx.orchestration.portable import outputs_utils
from tfx.orchestration.portable import python_driver_operator
from tfx.orchestration.portable import python_executor_operator
from tfx.orchestration.portable import resolver_node_handler
from tfx.orchestration.portable.input_resolution import exceptions
from tfx.orchestration.portable.mlmd import context_lib
from tfx.orchestration.portable.mlmd import execution_lib
from tfx.proto.orchestration import driver_output_pb2
from tfx.proto.orchestration import executable_spec_pb2
from tfx.proto.orchestration import execution_result_pb2
from tfx.proto.orchestration import pipeline_pb2
from tfx.utils import typing_utils

from google.protobuf import message
from ml_metadata.proto import metadata_store_pb2

# Subclasses of BaseExecutorOperator
ExecutorOperator = TypeVar(
    'ExecutorOperator', bound=base_executor_operator.BaseExecutorOperator)

# Subclasses of BaseDriverOperator
DriverOperator = TypeVar(
    'DriverOperator', bound=base_driver_operator.BaseDriverOperator)

DEFAULT_EXECUTOR_OPERATORS = {
    executable_spec_pb2.PythonClassExecutableSpec:
        python_executor_operator.PythonExecutorOperator,
    executable_spec_pb2.BeamExecutableSpec:
        beam_executor_operator.BeamExecutorOperator,
    executable_spec_pb2.ContainerExecutableSpec:
        docker_executor_operator.DockerExecutorOperator
}

DEFAULT_DRIVER_OPERATORS = {
    executable_spec_pb2.PythonClassExecutableSpec:
        python_driver_operator.PythonDriverOperator
}

_SYSTEM_NODE_HANDLERS = {
    'tfx.dsl.components.common.importer.Importer':
        importer_node_handler.ImporterNodeHandler,
    'tfx.dsl.components.common.resolver.Resolver':
        resolver_node_handler.ResolverNodeHandler,
}

_ERROR_CODE_UNIMPLEMENTED: int = grpc.StatusCode.UNIMPLEMENTED.value[0]

{'fraction_non_alphanumeric': 0.05223880597014925, 'fraction_numerical': 0.0052677787532923615, 'mean_word_length': 6.104989604989605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7305817', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1985, 'n_words': 534}