<commit_before>"""
VGSL plumbing
"""
import re
import json
import torch
import logging
import pathlib

from torch import nn
from typing import Sequence, List, Tuple, Union, Optional, Iterable, Callable, Dict, Any

from kraken.lib import layers
from kraken.lib.codec import PytorchCodec
from kraken.lib.exceptions import KrakenInvalidModelException

from coremltools.models import MLModel
from coremltools.models import datatypes
from coremltools.models.neural_network import NeuralNetworkBuilder<commit_msg>filter out non-pertinent coreml warnings<commit_after>"""
VGSL plumbing
"""
import re
import json
import torch
import logging
import warnings
import pathlib

from torch import nn
from typing import Sequence, List, Tuple, Union, Optional, Iterable, Callable, Dict, Any

from kraken.lib import layers
from kraken.lib.codec import PytorchCodec
from kraken.lib.exceptions import KrakenInvalidModelException

# filter out coreml warnings coming from their conversion routines (which we don't use).
with warnings.catch_warnings():
    warnings.filterwarnings(action='ignore', message='has not been tested with coremltools')
    warnings.filterwarnings(action='ignore', message='is not supported')

    from coremltools.models import MLModel
    from coremltools.models import datatypes
    from coremltools.models.neural_network import NeuralNetworkBuilder
{'fraction_non_alphanumeric': 0.0604274134119381, 'fraction_numerical': 0.0, 'mean_word_length': 6.147368421052631, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024244', 'n_tokens_mistral': 371, 'n_tokens_neox': 373, 'n_words': 137}