<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)


class PyTokenizers(PythonPackage):
    """Fast and Customizable Tokenizers."""

    homepage = "https://github.com/huggingface/tokenizers"
    url      = "https://pypi.io/packages/source/t/tokenizers/tokenizers-0.6.0.tar.gz"

    version('0.6.0', sha256='1da11fbfb4f73be695bed0d655576097d09a137a16dceab2f66399716afaffac')

    depends_on('py-setuptools', type='build')
    depends_on('py-setuptools-rust', type='build')<commit_msg>py-tokenizers: add 0.5.2 (#16039)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)


class PyTokenizers(PythonPackage):
    """Fast and Customizable Tokenizers."""

    homepage = "https://github.com/huggingface/tokenizers"
    url      = "https://pypi.io/packages/source/t/tokenizers/tokenizers-0.6.0.tar.gz"

    version('0.6.0', sha256='1da11fbfb4f73be695bed0d655576097d09a137a16dceab2f66399716afaffac')
    version('0.5.2', sha256='b5a235f9c71d04d4925df6c4fa13b13f1d03f9b7ac302b89f8120790c4f742bc')

    depends_on('py-setuptools', type='build')
    depends_on('py-setuptools-rust', type='build')
    depends_on('rust@nightly', type='build')

    # TODO: This package currently requires internet access to install.
    # Also, a nightly or dev version of rust is required to build.
    # https://github.com/huggingface/tokenizers/issues/176
    # https://github.com/PyO3/pyo3/issues/5
{'fraction_non_alphanumeric': 0.12868949232585597, 'fraction_numerical': 0.09917355371900827, 'mean_word_length': 4.631229235880399, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1561280', 'n_tokens_mistral': 690, 'n_tokens_neox': 600, 'n_words': 109}