<commit_before>import os
import shutil

from neuropods.backends.python.packager import create_python_neuropod

from ludwig.api import LudwigModel
from ludwig.globals import MODEL_HYPERPARAMETERS_FILE_NAME, \
    TRAIN_SET_METADATA_FILE_NAME, MODEL_WEIGHTS_FILE_NAME
from ludwig.utils.data_utils import load_json


class LudwigNeuropodModelWrapper:
    def __init__(self, data_root):
        self.ludwig_model = LudwigModel.load(data_root)

    def __call__(self, **kwargs):
        print('__call__', file=sys.stderr)
        return self.ludwig_model.predict(data_dict=kwargs, return_type=dict)


def get_model(data_root):
    print('get_model()', data_root, file=sys.stderr)
    return LudwigNeuropodModelWrapper(data_root)


def build_neuropod(
        ludwig_model_path,
        neuropod_path="neuropod"
):
    data_paths = [
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_HYPERPARAMETERS_FILE_NAME
        },
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_WEIGHTS_FILE_NAME + ".meta"
        },
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_WEIGHTS_FILE_NAME + ".index"
        },
        {
            "path": ludwig_model_path,
            "packaged_name": TRAIN_SET_METADATA_FILE_NAME
        },
    ]
    model_weights_files_pattern = MODEL_WEIGHTS_FILE_NAME + ".data-"
    for filename in os.listdir(ludwig_model_path):<commit_msg>Lazy loading of neuropods package<commit_after>import os
import shutil

from ludwig.api import LudwigModel
from ludwig.globals import MODEL_HYPERPARAMETERS_FILE_NAME, \
    TRAIN_SET_METADATA_FILE_NAME, MODEL_WEIGHTS_FILE_NAME
from ludwig.utils.data_utils import load_json


class LudwigNeuropodModelWrapper:
    def __init__(self, data_root):
        self.ludwig_model = LudwigModel.load(data_root)

    def __call__(self, **kwargs):
        print('__call__', file=sys.stderr)
        return self.ludwig_model.predict(data_dict=kwargs, return_type=dict)


def get_model(data_root):
    print('get_model()', data_root, file=sys.stderr)
    return LudwigNeuropodModelWrapper(data_root)


def build_neuropod(
        ludwig_model_path,
        neuropod_path="neuropod"
):
    from neuropods.backends.python.packager import create_python_neuropod

    data_paths = [
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_HYPERPARAMETERS_FILE_NAME
        },
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_WEIGHTS_FILE_NAME + ".meta"
        },
        {
            "path": ludwig_model_path,
            "packaged_name": MODEL_WEIGHTS_FILE_NAME + ".index"
        },
        {
            "path": ludwig_model_path,
            "packaged_name": TRAIN_SET_METADATA_FILE_NAME
        },
    ]
    model_weights_files_pattern = MODEL_WEIGHTS_FILE_NAME + ".data-"
    for filename in os.listdir(ludwig_model_path):
{'fraction_non_alphanumeric': 0.0861244019138756, 'fraction_numerical': 0.0, 'mean_word_length': 3.070931849791377, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2054062', 'n_tokens_mistral': 1018, 'n_tokens_neox': 970, 'n_words': 151}