<commit_before># -*- coding: utf-8 -*-
"""Objects representing MediaWiki families."""
#
# (C) Pywikibot team, 2004-2020
#
# Distributed under the terms of the MIT license.
#
import collections
import logging
import re
import string
import sys
import urllib.parse as urlparse
import warnings

from importlib import import_module
from itertools import chain
from os.path import basename, dirname, splitext

import pywikibot
from pywikibot.comms.http import fetch
from pywikibot import config
from pywikibot.exceptions import UnknownFamily, FamilyMaintenanceWarning
from pywikibot.tools import (
    deprecated, deprecated_args, remove_last_args, issue_deprecation_warning,
    ModuleDeprecationWrapper, FrozenDict, classproperty,
)


logger = logging.getLogger('pywiki.wiki.family')

# Legal characters for Family.name and Family.langs keys
NAME_CHARACTERS = string.ascii_letters + string.digits
# nds_nl code alias requires "_"n
# dash must be the last char to be reused as regex in update_linktrails
CODE_CHARACTERS = string.ascii_lowercase + string.digits + '_-'


class Family:

    """Parent singleton class for all wiki families."""

    def __new__(cls):
        """Allocator."""
        # any Family class defined in this file are abstract
        if cls in globals().values():
            raise TypeError(
                'Abstract Family class {0} cannot be instantiated; '
                'subclass it instead'.format(cls.__name__))

        # Override classproperty
        cls.instance = super().__new__(cls)
        # staticmethod is because python 2.7 binds the lambda to the class
        cls.__new__ = staticmethod(lambda cls: cls.instance)  # shortcut

        # don't use hasattr() here. consider only the class itself
        if '__init__' in cls.__dict__:
            # Initializer deprecated. Families should be immutable and any
            # instance / class modification should go to allocator (__new__).
            # The function is read from __dict__ because deprecated expect a
            # function and python 2.7 binds the method to the class.
            cls.__init__ = deprecated(cls.__dict__['__init__'])

            # Invoke initializer immediately and make initializer no-op.
            # This is to avoid repeated initializer invocation on repeated
            # invocations of the metaclass's __call__.
            cls.instance.__init__()
            cls.__init__ = lambda self: None  # no-op

        return cls.instance

    @classproperty
    def instance(cls):
        """Get the singleton instance."""
        # This is a placeholder to invoke allocator before it's allocated.
        # Allocator will override this classproperty.
        return cls()

    name = None

    langs = {}

    # For interwiki sorting order see
    # https://meta.wikimedia.org/wiki/Interwiki_sorting_order

    # The sorting order by language name from meta
    # MediaWiki:Interwiki_config-sorting_order-native-languagename
    alphabetic = [
        'ace', 'kbd', 'ady', 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an',
        'arc', 'roa-rup', 'frp', 'as', 'ast', 'atj', 'gn', 'av', 'ay', 'az',
        'bm', 'bn', 'bjn', 'zh-min-nan', 'nan', 'map-bms', 'ba', 'be',
        'be-tarask', 'bh', 'bcl', 'bi', 'bg', 'bar', 'bo', 'bs', 'br', 'bxr',
        'ca', 'cv', 'ceb', 'cs', 'ch', 'cbk-zam', 'ny', 'sn', 'tum', 'cho',
        'co', 'cy', 'da', 'dk', 'pdc', 'de', 'dv', 'nv', 'dsb', 'dty', 'dz',
        'mh', 'et', 'el', 'eml', 'en', 'myv', 'es', 'eo', 'ext', 'eu', 'ee',
        'fa', 'hif', 'fo', 'fr', 'fy', 'ff', 'fur', 'ga', 'gv', 'gag', 'gd',
        'gl', 'gan', 'ki', 'glk', 'gu', 'gor', 'got', 'hak', 'xal', 'ko', 'ha',
        'haw', 'hy', 'hi', 'ho', 'hsb', 'hr', 'hyw', 'io', 'ig', 'ilo', 'inh',
        'bpy', 'id', 'ia', 'ie', 'iu', 'ik', 'os', 'xh', 'zu', 'is', 'it',
        'he', 'jv', 'kbp', 'kl', 'kn', 'kr', 'pam', 'krc', 'ka', 'ks', 'csb',
        'kk', 'kw', 'rw', 'rn', 'sw', 'kv', 'kg', 'gom', 'ht', 'ku', 'kj',
        'ky', 'mrj', 'lad', 'lbe', 'lo', 'ltg', 'la', 'lv', 'lb', 'lez', 'lfn',
        'lt', 'lij', 'li', 'ln', 'olo', 'jbo', 'lg', 'lmo', 'lrc', 'hu', 'mai',
        'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'xmf', 'arz', 'mzn', 'ms', 'min',
        'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'na', 'fj', 'nl',
        'nds-nl', 'cr', 'ne', 'new', 'ja', 'nqo', 'nap', 'ce', 'frr', 'pih',
        'no', 'nb', 'nn', 'nrm', 'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng',
        'hz', 'uz', 'pa', 'pi', 'pfl', 'pag', 'pnb', 'pap', 'ps', 'jam', 'koi',
        'km', 'pcd', 'pms', 'tpi', 'nds', 'pl', 'pnt', 'pt', 'aa', 'kaa',
        'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'rue', 'ru', 'sah', 'se',
        'sm', 'sa', 'sg', 'sat', 'sc', 'sco', 'stq', 'st', 'nso', 'tn', 'sq',
        'scn', 'si', 'simple', 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so',
        'ckb', 'srn', 'sr', 'sh', 'su', 'fi', 'sv', 'tl', 'shn', 'ta', 'kab',
        'roa-tara', 'tt', 'te', 'tet', 'th', 'ti', 'tg', 'to', 'chr', 'chy',
        've', 'tcy', 'tr', 'azb', 'tk', 'tw', 'tyv', 'din', 'udm', 'bug', 'uk',
        'ur', 'ug', 'za', 'vec', 'vep', 'vi', 'vo', 'fiu-vro', 'wa',
        'zh-classical', 'vls', 'war', 'wo', 'wuu', 'ts', 'yi', 'yo', 'zh-yue',
        'diq', 'zea', 'bat-smg', 'zh', 'zh-tw', 'zh-cn'
    ]

    # The revised sorting order by first word from meta
    # MediaWiki:Interwiki_config-sorting_order-native-languagename-firstword
    alphabetic_revised = [
        'ace', 'ady', 'kbd', 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an',
        'arc', 'roa-rup', 'frp', 'as', 'ast', 'atj', 'gn', 'av', 'ay', 'az',
        'bjn', 'id', 'ms', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'jv',
        'su', 'ba', 'min', 'be', 'be-tarask', 'bh', 'bcl', 'bi', 'bar', 'bo',
        'bs', 'br', 'bug', 'bg', 'bxr', 'ca', 'ceb', 'cv', 'cs', 'ch',
        'cbk-zam', 'ny', 'sn', 'tum', 'cho', 'co', 'cy', 'da', 'dk', 'pdc',
        'de', 'dv', 'nv', 'dsb', 'na', 'dty', 'dz', 'mh', 'et', 'el', 'eml',
        'en', 'myv', 'es', 'eo', 'ext', 'eu', 'ee', 'fa', 'hif', 'fo', 'fr',
        'fy', 'ff', 'fur', 'ga', 'gv', 'sm', 'gag', 'gd', 'gl', 'gan', 'ki',
        'glk', 'gu', 'got', 'hak', 'xal', 'ko', 'ha', 'haw', 'hy', 'hi', 'ho',
        'hsb', 'hr', 'hyw', 'io', 'ig', 'ilo', 'inh', 'bpy', 'ia', 'ie', 'iu',
        'ik', 'os', 'xh', 'zu', 'is', 'it', 'he', 'kl', 'kn', 'kr', 'pam',
        'ka', 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'mrj', 'sw', 'kv',
        'kg', 'gom', 'gor', 'ht', 'ku', 'shn', 'kj', 'lad', 'lbe', 'lez',
        'lfn', 'lo', 'la', 'ltg', 'lv', 'to', 'lb', 'lt', 'lij', 'li', 'ln',
        'olo', 'jbo', 'lg', 'lmo', 'lrc', 'hu', 'mai', 'mk', 'mg', 'ml', 'krc',
        'mt', 'mi', 'mr', 'xmf', 'arz', 'mzn', 'cdo', 'mwl', 'koi', 'mdf',
        'mo', 'mn', 'mus', 'my', 'nah', 'fj', 'nl', 'nds-nl', 'cr', 'ne',
        'new', 'ja', 'nqo', 'nap', 'ce', 'frr', 'pih', 'no', 'nb', 'nn', 'nrm',
        'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pi',
        'pfl', 'pag', 'pnb', 'pap', 'ps', 'jam', 'km', 'pcd', 'pms', 'nds',
        'pl', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm',
        'qu', 'ru', 'rue', 'sah', 'se', 'sa', 'sg', 'sat', 'sc', 'sco', 'stq',
        'st', 'nso', 'tn', 'sq', 'scn', 'si', 'simple', 'sd', 'ss', 'sk', 'sl',
        'cu', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'fi', 'sv', 'tl', 'ta',
        'kab', 'kbp', 'roa-tara', 'tt', 'te', 'tet', 'th', 'vi', 'ti', 'tg',
        'tpi', 'chr', 'chy', 've', 'tcy', 'tr', 'azb', 'tk', 'tw', 'tyv',
        'din', 'udm', 'uk', 'ur', 'ug', 'za', 'vec', 'vep', 'vo', 'fiu-vro',
        'wa', 'zh-classical', 'vls', 'war', 'wo', 'wuu', 'ts', 'yi', 'yo',
        'zh-yue', 'diq', 'zea', 'bat-smg', 'zh', 'zh-tw', 'zh-cn'
    ]

    # Order for fy: alphabetical by code, but y counts as i
    fyinterwiki = alphabetic[:]
    fyinterwiki.remove('nb')
    fyinterwiki.sort(key=lambda x:
                     x.replace('y', 'i') + x.count('y') * '!')

    # Letters that can follow a wikilink and are regarded as part of
    # this link. This depends on the linktrail setting in LanguageXx.php
    #
    # Do not use this dict directly but Site.linktrail or Family.linktrail
    # methods instead
    linktrails = {
        '_default': '[a-z]*',
        'ab': '[a-zабвгҕдежзӡикқҟлмнопҧрстҭуфхҳцҵчҷҽҿшыҩџьә]*',
        'als': '[äöüßa-z]*',
        'an': '[a-záéíóúñ]*',
        'ar': '[a-zء-ي]*',
        'arz': '[a-zء-ي]*',
        'ast': '[a-záéíóúñ]*',
        'atj': '[a-zàâçéèêîôûäëïöüùÇÉÂÊÎÔÛÄËÏÖÜÀÈÙ]*',
        'av': '[a-zабвгдеёжзийклмнопрстуфхцчшщъыьэюя]*',
        'awa': '[a-zऀ-ॣ०-꣠-ꣿ]*',
        'ay': '[a-záéíóúñ]*',
        'az': '[a-zçəğıöşü]*',
        'azb': '[ابپتثجچحخدذرزژسشصضطظعغفقکگلمنوهیآأئؤة‌]*',
        'ba': '[a-zабвгдеёжзийклмнопрстуфхцчшщъыьэюяәөүғҡңҙҫһ“»]*',
        'bar': '[äöüßa-z]*',
        'bat-smg': '[a-ząčęėįšųūž]*',
        'be': '[абвгґджзеёжзійклмнопрстуўфхцчшыьэюяćčłńśšŭźža-z]*',<commit_msg>[L10N] Update linktrails<commit_after># -*- coding: utf-8 -*-
"""Objects representing MediaWiki families."""
#
# (C) Pywikibot team, 2004-2020
#
# Distributed under the terms of the MIT license.
#
import collections
import logging
import re
import string
import sys
import urllib.parse as urlparse
import warnings

from importlib import import_module
from itertools import chain
from os.path import basename, dirname, splitext

import pywikibot
from pywikibot.comms.http import fetch
from pywikibot import config
from pywikibot.exceptions import UnknownFamily, FamilyMaintenanceWarning
from pywikibot.tools import (
    deprecated, deprecated_args, remove_last_args, issue_deprecation_warning,
    ModuleDeprecationWrapper, FrozenDict, classproperty,
)


logger = logging.getLogger('pywiki.wiki.family')

# Legal characters for Family.name and Family.langs keys
NAME_CHARACTERS = string.ascii_letters + string.digits
# nds_nl code alias requires "_"n
# dash must be the last char to be reused as regex in update_linktrails
CODE_CHARACTERS = string.ascii_lowercase + string.digits + '_-'


class Family:

    """Parent singleton class for all wiki families."""

    def __new__(cls):
        """Allocator."""
        # any Family class defined in this file are abstract
        if cls in globals().values():
            raise TypeError(
                'Abstract Family class {0} cannot be instantiated; '
                'subclass it instead'.format(cls.__name__))

        # Override classproperty
        cls.instance = super().__new__(cls)
        # staticmethod is because python 2.7 binds the lambda to the class
        cls.__new__ = staticmethod(lambda cls: cls.instance)  # shortcut

        # don't use hasattr() here. consider only the class itself
        if '__init__' in cls.__dict__:
            # Initializer deprecated. Families should be immutable and any
            # instance / class modification should go to allocator (__new__).
            # The function is read from __dict__ because deprecated expect a
            # function and python 2.7 binds the method to the class.
            cls.__init__ = deprecated(cls.__dict__['__init__'])

            # Invoke initializer immediately and make initializer no-op.
            # This is to avoid repeated initializer invocation on repeated
            # invocations of the metaclass's __call__.
            cls.instance.__init__()
            cls.__init__ = lambda self: None  # no-op

        return cls.instance

    @classproperty
    def instance(cls):
        """Get the singleton instance."""
        # This is a placeholder to invoke allocator before it's allocated.
        # Allocator will override this classproperty.
        return cls()

    name = None

    langs = {}

    # For interwiki sorting order see
    # https://meta.wikimedia.org/wiki/Interwiki_sorting_order

    # The sorting order by language name from meta
    # MediaWiki:Interwiki_config-sorting_order-native-languagename
    alphabetic = [
        'ace', 'kbd', 'ady', 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an',
        'arc', 'roa-rup', 'frp', 'as', 'ast', 'atj', 'gn', 'av', 'ay', 'az',
        'bm', 'bn', 'bjn', 'zh-min-nan', 'nan', 'map-bms', 'ba', 'be',
        'be-tarask', 'bh', 'bcl', 'bi', 'bg', 'bar', 'bo', 'bs', 'br', 'bxr',
        'ca', 'cv', 'ceb', 'cs', 'ch', 'cbk-zam', 'ny', 'sn', 'tum', 'cho',
        'co', 'cy', 'da', 'dk', 'pdc', 'de', 'dv', 'nv', 'dsb', 'dty', 'dz',
        'mh', 'et', 'el', 'eml', 'en', 'myv', 'es', 'eo', 'ext', 'eu', 'ee',
        'fa', 'hif', 'fo', 'fr', 'fy', 'ff', 'fur', 'ga', 'gv', 'gag', 'gd',
        'gl', 'gan', 'ki', 'glk', 'gu', 'gor', 'got', 'hak', 'xal', 'ko', 'ha',
        'haw', 'hy', 'hi', 'ho', 'hsb', 'hr', 'hyw', 'io', 'ig', 'ilo', 'inh',
        'bpy', 'id', 'ia', 'ie', 'iu', 'ik', 'os', 'xh', 'zu', 'is', 'it',
        'he', 'jv', 'kbp', 'kl', 'kn', 'kr', 'pam', 'krc', 'ka', 'ks', 'csb',
        'kk', 'kw', 'rw', 'rn', 'sw', 'kv', 'kg', 'gom', 'ht', 'ku', 'kj',
        'ky', 'mrj', 'lad', 'lbe', 'lo', 'ltg', 'la', 'lv', 'lb', 'lez', 'lfn',
        'lt', 'lij', 'li', 'ln', 'olo', 'jbo', 'lg', 'lmo', 'lrc', 'hu', 'mai',
        'mk', 'mg', 'ml', 'mt', 'mi', 'mr', 'xmf', 'arz', 'mzn', 'ms', 'min',
        'cdo', 'mwl', 'mdf', 'mo', 'mn', 'mus', 'my', 'nah', 'na', 'fj', 'nl',
        'nds-nl', 'cr', 'ne', 'new', 'ja', 'nqo', 'nap', 'ce', 'frr', 'pih',
        'no', 'nb', 'nn', 'nrm', 'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng',
        'hz', 'uz', 'pa', 'pi', 'pfl', 'pag', 'pnb', 'pap', 'ps', 'jam', 'koi',
        'km', 'pcd', 'pms', 'tpi', 'nds', 'pl', 'pnt', 'pt', 'aa', 'kaa',
        'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm', 'qu', 'rue', 'ru', 'sah', 'se',
        'sm', 'sa', 'sg', 'sat', 'sc', 'sco', 'stq', 'st', 'nso', 'tn', 'sq',
        'scn', 'si', 'simple', 'sd', 'ss', 'sk', 'sl', 'cu', 'szl', 'so',
        'ckb', 'srn', 'sr', 'sh', 'su', 'fi', 'sv', 'tl', 'shn', 'ta', 'kab',
        'roa-tara', 'tt', 'te', 'tet', 'th', 'ti', 'tg', 'to', 'chr', 'chy',
        've', 'tcy', 'tr', 'azb', 'tk', 'tw', 'tyv', 'din', 'udm', 'bug', 'uk',
        'ur', 'ug', 'za', 'vec', 'vep', 'vi', 'vo', 'fiu-vro', 'wa',
        'zh-classical', 'vls', 'war', 'wo', 'wuu', 'ts', 'yi', 'yo', 'zh-yue',
        'diq', 'zea', 'bat-smg', 'zh', 'zh-tw', 'zh-cn'
    ]

    # The revised sorting order by first word from meta
    # MediaWiki:Interwiki_config-sorting_order-native-languagename-firstword
    alphabetic_revised = [
        'ace', 'ady', 'kbd', 'af', 'ak', 'als', 'am', 'ang', 'ab', 'ar', 'an',
        'arc', 'roa-rup', 'frp', 'as', 'ast', 'atj', 'gn', 'av', 'ay', 'az',
        'bjn', 'id', 'ms', 'bm', 'bn', 'zh-min-nan', 'nan', 'map-bms', 'jv',
        'su', 'ba', 'min', 'be', 'be-tarask', 'bh', 'bcl', 'bi', 'bar', 'bo',
        'bs', 'br', 'bug', 'bg', 'bxr', 'ca', 'ceb', 'cv', 'cs', 'ch',
        'cbk-zam', 'ny', 'sn', 'tum', 'cho', 'co', 'cy', 'da', 'dk', 'pdc',
        'de', 'dv', 'nv', 'dsb', 'na', 'dty', 'dz', 'mh', 'et', 'el', 'eml',
        'en', 'myv', 'es', 'eo', 'ext', 'eu', 'ee', 'fa', 'hif', 'fo', 'fr',
        'fy', 'ff', 'fur', 'ga', 'gv', 'sm', 'gag', 'gd', 'gl', 'gan', 'ki',
        'glk', 'gu', 'got', 'hak', 'xal', 'ko', 'ha', 'haw', 'hy', 'hi', 'ho',
        'hsb', 'hr', 'hyw', 'io', 'ig', 'ilo', 'inh', 'bpy', 'ia', 'ie', 'iu',
        'ik', 'os', 'xh', 'zu', 'is', 'it', 'he', 'kl', 'kn', 'kr', 'pam',
        'ka', 'ks', 'csb', 'kk', 'kw', 'rw', 'ky', 'rn', 'mrj', 'sw', 'kv',
        'kg', 'gom', 'gor', 'ht', 'ku', 'shn', 'kj', 'lad', 'lbe', 'lez',
        'lfn', 'lo', 'la', 'ltg', 'lv', 'to', 'lb', 'lt', 'lij', 'li', 'ln',
        'olo', 'jbo', 'lg', 'lmo', 'lrc', 'hu', 'mai', 'mk', 'mg', 'ml', 'krc',
        'mt', 'mi', 'mr', 'xmf', 'arz', 'mzn', 'cdo', 'mwl', 'koi', 'mdf',
        'mo', 'mn', 'mus', 'my', 'nah', 'fj', 'nl', 'nds-nl', 'cr', 'ne',
        'new', 'ja', 'nqo', 'nap', 'ce', 'frr', 'pih', 'no', 'nb', 'nn', 'nrm',
        'nov', 'ii', 'oc', 'mhr', 'or', 'om', 'ng', 'hz', 'uz', 'pa', 'pi',
        'pfl', 'pag', 'pnb', 'pap', 'ps', 'jam', 'km', 'pcd', 'pms', 'nds',
        'pl', 'pnt', 'pt', 'aa', 'kaa', 'crh', 'ty', 'ksh', 'ro', 'rmy', 'rm',
        'qu', 'ru', 'rue', 'sah', 'se', 'sa', 'sg', 'sat', 'sc', 'sco', 'stq',
        'st', 'nso', 'tn', 'sq', 'scn', 'si', 'simple', 'sd', 'ss', 'sk', 'sl',
        'cu', 'szl', 'so', 'ckb', 'srn', 'sr', 'sh', 'fi', 'sv', 'tl', 'ta',
        'kab', 'kbp', 'roa-tara', 'tt', 'te', 'tet', 'th', 'vi', 'ti', 'tg',
        'tpi', 'chr', 'chy', 've', 'tcy', 'tr', 'azb', 'tk', 'tw', 'tyv',
        'din', 'udm', 'uk', 'ur', 'ug', 'za', 'vec', 'vep', 'vo', 'fiu-vro',
        'wa', 'zh-classical', 'vls', 'war', 'wo', 'wuu', 'ts', 'yi', 'yo',
        'zh-yue', 'diq', 'zea', 'bat-smg', 'zh', 'zh-tw', 'zh-cn'
    ]

    # Order for fy: alphabetical by code, but y counts as i
    fyinterwiki = alphabetic[:]
    fyinterwiki.remove('nb')
    fyinterwiki.sort(key=lambda x:
                     x.replace('y', 'i') + x.count('y') * '!')

    # Letters that can follow a wikilink and are regarded as part of
    # this link. This depends on the linktrail setting in LanguageXx.php
    #
    # Do not use this dict directly but Site.linktrail or Family.linktrail
    # methods instead
    linktrails = {
        '_default': '[a-z]*',
        'ab': '[a-zабвгҕдежзӡикқҟлмнопҧрстҭуфхҳцҵчҷҽҿшыҩџьә]*',
        'als': '[äöüßa-z]*',
        'an': '[a-záéíóúñ]*',
        'ar': '[a-zء-ي]*',
        'ary': '[a-zء-ي]*',
        'arz': '[a-zء-ي]*',
        'ast': '[a-záéíóúñ]*',
        'atj': '[a-zàâçéèêîôûäëïöüùÇÉÂÊÎÔÛÄËÏÖÜÀÈÙ]*',
        'av': '[a-zабвгдеёжзийклмнопрстуфхцчшщъыьэюя]*',
        'avk': '[a-zàâçéèêîôûäëïöüùÇÉÂÊÎÔÛÄËÏÖÜÀÈÙ]*',
        'awa': '[a-zऀ-ॣ०-꣠-ꣿ]*',
        'ay': '[a-záéíóúñ]*',
        'az': '[a-zçəğıöşü]*',
        'azb': '[ابپتثجچحخدذرزژسشصضطظعغفقکگلمنوهیآأئؤة‌]*',
        'ba': '[a-zабвгдеёжзийклмнопрстуфхцчшщъыьэюяәөүғҡңҙҫһ“»]*',
        'bar': '[äöüßa-z]*',
        'bat-smg': '[a-ząčęėįšųūž]*',
        'be': '[абвгґджзеёжзійклмнопрстуўфхцчшыьэюяćčłńśšŭźža-z]*',
{'fraction_non_alphanumeric': 0.2624164810690423, 'fraction_numerical': 0.0017817371937639199, 'mean_word_length': 2.973672566371681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1261878', 'n_tokens_mistral': 7604, 'n_tokens_neox': 7501, 'n_words': 2025}