<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-#
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bleeding-edge version of Unicode Character Database.

Provides an interface similar to Python's own unicodedata package, but with
the bleeding-edge data. The implementation is not efficient at all, it's
just done this way for the ease of use. The data is coming from bleeding
edge version of the Unicode Standard not yet published, so it is expected to
be unstable and sometimes inconsistent.
"""

__author__ = (
    "<EMAIL> (<NAME>) and "
    "<EMAIL> (<NAME>)")

import codecs
import collections
import os
from os import path
import re
import sys

from fontTools.misc.py23 import unichr
try:
  import unicodedata2 as unicodedata  # Unicode 8 compliant native lib
except ImportError:
  import unicodedata  # Python's internal library

from nototools import tool_utils # parse_int_ranges

_data_is_loaded = False
_property_value_aliases_data = {}
_character_names_data = {}
_general_category_data = {}
_combining_class_data = {}
_decomposition_data = {}
_bidi_mirroring_characters = set()
_script_data = {}
_script_extensions_data = {}
_block_data = {}
_block_range = {}
_block_names = []
_age_data = {}
_bidi_mirroring_glyph_data = {}
_core_properties_data = {}
_indic_positional_data = {}
_indic_syllabic_data = {}
_defined_characters = set()
_script_code_to_long_name = {}
_folded_script_name_to_code = {}
_lower_to_upper_case = {}

# emoji data
_presentation_default_emoji = None
_presentation_default_text = None
_emoji_modifier_base = None
_emoji = None
_emoji_variants = None
_emoji_variants_proposed = None

# non-emoji variant data
_variant_data = None
_variant_data_cps = None

# proposed emoji
_proposed_emoji_data = None
_proposed_emoji_data_cps = None

# emoji sequences
_emoji_sequence_data = None
_emoji_non_vs_to_canonical = None

# nameslist/namealiases
_nameslist_see_also = None
_namealiases_alt_names = None

def load_data():
  """Loads the data files needed for the module.

  Could be used by processes that care about controlling when the data is
  loaded. Otherwise, data will be loaded the first time it's needed.
  """
  global _data_is_loaded

  if not _data_is_loaded:
    _load_property_value_aliases_txt()
    _load_unicode_data_txt()
    _load_scripts_txt()
    _load_script_extensions_txt()
    _load_blocks_txt()
    _load_derived_age_txt()
    _load_derived_core_properties_txt()
    _load_bidi_mirroring_txt()
    _load_indic_data()
    _load_emoji_data()
    _load_emoji_sequence_data()
    _load_unicode_emoji_variants()
    _load_variant_data()
    _load_proposed_emoji_data()
    _load_nameslist_data()
    _load_namealiases_data()

    _data_is_loaded = True


def name(char, *args):
  """Returns the name of a character.

  Raises a ValueError exception if the character is undefined, unless an
  extra argument is given, in which case it will return that argument.
  """
  if type(char) is int:
    char = unichr(char)
  # First try and get the name from unidata, which is faster and supports
  # CJK and Hangul automatic names
  try:
      return unicodedata.name(char)
  except ValueError as val_error:
    cp = ord(char)
    load_data()
    if cp in _character_names_data:
      return _character_names_data[cp]
    elif (cp,) in _emoji_sequence_data:
      return _emoji_sequence_data[(cp,)][0]
    elif args:
      return args[0]
    else:
      raise Exception('no name for "%0x"' % ord(char))


def _char_to_int(char):
  """Converts a potential character to its scalar value."""
  if type(char) in [str, unicode]:
    return ord(char)
  else:
    return char

def derived_props():
  load_data()
  return frozenset(_core_properties_data.keys())

def chars_with_property(propname):
  load_data()
  return frozenset(_core_properties_data[propname])

def category(char):
  """Returns the general category of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _general_category_data[char]
  except KeyError:
    return "Cn"  # Unassigned


def combining(char):
  """Returns the canonical combining class of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _combining_class_data[char]
  except KeyError:
    return 0


def to_upper(char):
  """Returns the upper case for a lower case character.
  This is not full upper casing, but simply reflects the 1-1
  mapping in UnicodeData.txt."""
  load_data()
  cp = _char_to_int(char)
  try:
    if _general_category_data[cp] == 'Ll':
      return unichr(_lower_to_upper_case[cp])
  except KeyError:
    pass
  return char


def canonical_decomposition(char):
  """Returns the canonical decomposition of a character as a Unicode string.
  """
  load_data()
  char = _char_to_int(char)
  try:
    return _decomposition_data[char]
  except KeyError:
    return u""


def script(char):
  """Returns the script property of a character as a four-letter code."""
  load_data()
  char = _char_to_int(char)
  try:
    return _script_data[char]
  except KeyError:
    return "Zzzz"  # Unknown


def script_extensions(char):
  """Returns the script extensions property of a character.

  The return value is a frozenset of four-letter script codes.
  """
  load_data()
  char = _char_to_int(char)
  try:
    return _script_extensions_data[char]
  except KeyError:
    return frozenset([script(char)])


def block(char):
  """Returns the block property of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _block_data[char]
  except KeyError:
    return "No_Block"


def block_range(block):
  """Returns a range (first, last) of the named block."""
  load_data()
  return _block_range[block]


def block_chars(block):
  """Returns a frozenset of the cps in the named block."""
  load_data()
  first, last = _block_range[block]
  return frozenset(xrange(first, last + 1))


def block_names():
  """Returns the names of the blocks in block order."""
  load_data()
  return _block_names[:]


def age(char):
  """Returns the age property of a character as a string.

  Returns None if the character is unassigned."""
  load_data()
  char = _char_to_int(char)
  try:
    return _age_data[char]
  except KeyError:
    return None


# Uniscribe treats these ignorables (Hangul fillers) as spacing.
UNISCRIBE_USED_IGNORABLES = frozenset([0x115f, 0x1160, 0x3164, 0xffa0])

def is_default_ignorable(char):
  """Returns true if the character has the Default_Ignorable property."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return char in _core_properties_data["Default_Ignorable_Code_Point"]

def default_ignorables():
  load_data()
  return frozenset(_core_properties_data["Default_Ignorable_Code_Point"])


def is_defined(char):
  """Returns true if the character is defined in the Unicode Standard."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return char in _defined_characters


def is_private_use(char):
  """Returns true if the characters is a private use character."""
  return category(char) == "Co"


def mirrored(char):
  """Returns 1 if the characters is bidi mirroring, 0 otherwise."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return int(char in _bidi_mirroring_characters)


def bidi_mirroring_glyph(char):
  """Returns the bidi mirroring glyph property of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _bidi_mirroring_glyph_data[char]
  except KeyError:
    return None


def mirrored_chars():
  return frozenset(_bidi_mirroring_glyph_data.keys())


def indic_positional_category(char):
  """Returns the Indic positional category of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _indic_positional_data[char]
  except KeyError:
    return "NA"


def indic_syllabic_category(char):
  """Returns the Indic syllabic category of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _bidi_syllabic_data[char]
  except KeyError:
    return "Other"


def create_script_to_chars():
  """Returns a mapping from script to defined characters, based on script and
  extensions, for all scripts."""
  load_data()
  result = collections.defaultdict(set)
  for cp in _defined_characters:
    if cp in _script_data:
      result[_script_data[cp]].add(cp)
    if cp in _script_extensions_data:
      for script in _script_extensions_data[cp]:
        result[script].add(cp)
  return result


_DEFINED_CHARACTERS_CACHE = {}

def defined_characters(version=None, scr=None):
  """Returns the set of all defined characters in the Unicode Standard."""
  load_data()
  # handle common error where version is passed as string, the age test
  # will always pass
  if version is not None:
    version = float(version)
  try:
    return _DEFINED_CHARACTERS_CACHE[(version, scr)]
  except KeyError:
    pass
  characters = _defined_characters
  if version is not None:
    characters = {char for char in characters
                  if age(char) is not None and float(age(char)) <= version}
  if scr is not None:
    characters = {char for char in characters
                  if script(char) == scr or scr in script_extensions(char)}
  characters = frozenset(characters)
  _DEFINED_CHARACTERS_CACHE[(version, scr)] = characters
  return characters


_strip_re = re.compile(r"['-_ ]+")
def _folded_script_name(script_name):
  """Folds a script name to its bare bones for comparison."""
  # string.translate is changed by codecs, the method no longer takes two
  # parameters and so script_name.translate(None, "'-_ ") fails to compile
  return _strip_re.sub(script_name, '')


def script_code(script_name):
  """Returns the four-letter ISO 15924 code of a script from its long name.
  """
  load_data()
  folded_script_name = _folded_script_name(script_name)
  try:
    return _HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE[folded_script_name]
  except:
    return _folded_script_name_to_code.get(folded_script_name, 'Zzzz')


_HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES = {
    'Aran': 'Nastaliq', # not assigned
    'Nkoo': 'N\'Ko',
    'Phag': 'Phags-Pa',
    'Piqd': 'Klingon', # not assigned
    'Zmth': 'Math', # not assigned
    'Zsye': 'Emoji', # not assigned
    'Zsym': 'Symbols', # not assigned
}

_HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE = {
    _folded_script_name(name): code for code, name in
    _HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES.iteritems()
}
<commit_msg>Fix errors in folded_script_name.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-#
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bleeding-edge version of Unicode Character Database.

Provides an interface similar to Python's own unicodedata package, but with
the bleeding-edge data. The implementation is not efficient at all, it's
just done this way for the ease of use. The data is coming from bleeding
edge version of the Unicode Standard not yet published, so it is expected to
be unstable and sometimes inconsistent.
"""

__author__ = (
    "<EMAIL> (<NAME>) and "
    "<EMAIL> (<NAME>)")

import codecs
import collections
import os
from os import path
import re
import sys

from fontTools.misc.py23 import unichr
try:
  import unicodedata2 as unicodedata  # Unicode 8 compliant native lib
except ImportError:
  import unicodedata  # Python's internal library

from nototools import tool_utils # parse_int_ranges

_data_is_loaded = False
_property_value_aliases_data = {}
_character_names_data = {}
_general_category_data = {}
_combining_class_data = {}
_decomposition_data = {}
_bidi_mirroring_characters = set()
_script_data = {}
_script_extensions_data = {}
_block_data = {}
_block_range = {}
_block_names = []
_age_data = {}
_bidi_mirroring_glyph_data = {}
_core_properties_data = {}
_indic_positional_data = {}
_indic_syllabic_data = {}
_defined_characters = set()
_script_code_to_long_name = {}
_folded_script_name_to_code = {}
_lower_to_upper_case = {}

# emoji data
_presentation_default_emoji = None
_presentation_default_text = None
_emoji_modifier_base = None
_emoji = None
_emoji_variants = None
_emoji_variants_proposed = None

# non-emoji variant data
_variant_data = None
_variant_data_cps = None

# proposed emoji
_proposed_emoji_data = None
_proposed_emoji_data_cps = None

# emoji sequences
_emoji_sequence_data = None
_emoji_non_vs_to_canonical = None

# nameslist/namealiases
_nameslist_see_also = None
_namealiases_alt_names = None

def load_data():
  """Loads the data files needed for the module.

  Could be used by processes that care about controlling when the data is
  loaded. Otherwise, data will be loaded the first time it's needed.
  """
  global _data_is_loaded

  if not _data_is_loaded:
    _load_property_value_aliases_txt()
    _load_unicode_data_txt()
    _load_scripts_txt()
    _load_script_extensions_txt()
    _load_blocks_txt()
    _load_derived_age_txt()
    _load_derived_core_properties_txt()
    _load_bidi_mirroring_txt()
    _load_indic_data()
    _load_emoji_data()
    _load_emoji_sequence_data()
    _load_unicode_emoji_variants()
    _load_variant_data()
    _load_proposed_emoji_data()
    _load_nameslist_data()
    _load_namealiases_data()

    _data_is_loaded = True


def name(char, *args):
  """Returns the name of a character.

  Raises a ValueError exception if the character is undefined, unless an
  extra argument is given, in which case it will return that argument.
  """
  if type(char) is int:
    char = unichr(char)
  # First try and get the name from unidata, which is faster and supports
  # CJK and Hangul automatic names
  try:
      return unicodedata.name(char)
  except ValueError as val_error:
    cp = ord(char)
    load_data()
    if cp in _character_names_data:
      return _character_names_data[cp]
    elif (cp,) in _emoji_sequence_data:
      return _emoji_sequence_data[(cp,)][0]
    elif args:
      return args[0]
    else:
      raise Exception('no name for "%0x"' % ord(char))


def _char_to_int(char):
  """Converts a potential character to its scalar value."""
  if type(char) in [str, unicode]:
    return ord(char)
  else:
    return char

def derived_props():
  load_data()
  return frozenset(_core_properties_data.keys())

def chars_with_property(propname):
  load_data()
  return frozenset(_core_properties_data[propname])

def category(char):
  """Returns the general category of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _general_category_data[char]
  except KeyError:
    return "Cn"  # Unassigned


def combining(char):
  """Returns the canonical combining class of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _combining_class_data[char]
  except KeyError:
    return 0


def to_upper(char):
  """Returns the upper case for a lower case character.
  This is not full upper casing, but simply reflects the 1-1
  mapping in UnicodeData.txt."""
  load_data()
  cp = _char_to_int(char)
  try:
    if _general_category_data[cp] == 'Ll':
      return unichr(_lower_to_upper_case[cp])
  except KeyError:
    pass
  return char


def canonical_decomposition(char):
  """Returns the canonical decomposition of a character as a Unicode string.
  """
  load_data()
  char = _char_to_int(char)
  try:
    return _decomposition_data[char]
  except KeyError:
    return u""


def script(char):
  """Returns the script property of a character as a four-letter code."""
  load_data()
  char = _char_to_int(char)
  try:
    return _script_data[char]
  except KeyError:
    return "Zzzz"  # Unknown


def script_extensions(char):
  """Returns the script extensions property of a character.

  The return value is a frozenset of four-letter script codes.
  """
  load_data()
  char = _char_to_int(char)
  try:
    return _script_extensions_data[char]
  except KeyError:
    return frozenset([script(char)])


def block(char):
  """Returns the block property of a character."""
  load_data()
  char = _char_to_int(char)
  try:
    return _block_data[char]
  except KeyError:
    return "No_Block"


def block_range(block):
  """Returns a range (first, last) of the named block."""
  load_data()
  return _block_range[block]


def block_chars(block):
  """Returns a frozenset of the cps in the named block."""
  load_data()
  first, last = _block_range[block]
  return frozenset(xrange(first, last + 1))


def block_names():
  """Returns the names of the blocks in block order."""
  load_data()
  return _block_names[:]


def age(char):
  """Returns the age property of a character as a string.

  Returns None if the character is unassigned."""
  load_data()
  char = _char_to_int(char)
  try:
    return _age_data[char]
  except KeyError:
    return None


# Uniscribe treats these ignorables (Hangul fillers) as spacing.
UNISCRIBE_USED_IGNORABLES = frozenset([0x115f, 0x1160, 0x3164, 0xffa0])

def is_default_ignorable(char):
  """Returns true if the character has the Default_Ignorable property."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return char in _core_properties_data["Default_Ignorable_Code_Point"]

def default_ignorables():
  load_data()
  return frozenset(_core_properties_data["Default_Ignorable_Code_Point"])


def is_defined(char):
  """Returns true if the character is defined in the Unicode Standard."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return char in _defined_characters


def is_private_use(char):
  """Returns true if the characters is a private use character."""
  return category(char) == "Co"


def mirrored(char):
  """Returns 1 if the characters is bidi mirroring, 0 otherwise."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  return int(char in _bidi_mirroring_characters)


def bidi_mirroring_glyph(char):
  """Returns the bidi mirroring glyph property of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _bidi_mirroring_glyph_data[char]
  except KeyError:
    return None


def mirrored_chars():
  return frozenset(_bidi_mirroring_glyph_data.keys())


def indic_positional_category(char):
  """Returns the Indic positional category of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _indic_positional_data[char]
  except KeyError:
    return "NA"


def indic_syllabic_category(char):
  """Returns the Indic syllabic category of a character."""
  load_data()
  if type(char) in [str, unicode]:
    char = ord(char)
  try:
    return _bidi_syllabic_data[char]
  except KeyError:
    return "Other"


def create_script_to_chars():
  """Returns a mapping from script to defined characters, based on script and
  extensions, for all scripts."""
  load_data()
  result = collections.defaultdict(set)
  for cp in _defined_characters:
    if cp in _script_data:
      result[_script_data[cp]].add(cp)
    if cp in _script_extensions_data:
      for script in _script_extensions_data[cp]:
        result[script].add(cp)
  return result


_DEFINED_CHARACTERS_CACHE = {}

def defined_characters(version=None, scr=None):
  """Returns the set of all defined characters in the Unicode Standard."""
  load_data()
  # handle common error where version is passed as string, the age test
  # will always pass
  if version is not None:
    version = float(version)
  try:
    return _DEFINED_CHARACTERS_CACHE[(version, scr)]
  except KeyError:
    pass
  characters = _defined_characters
  if version is not None:
    characters = {char for char in characters
                  if age(char) is not None and float(age(char)) <= version}
  if scr is not None:
    characters = {char for char in characters
                  if script(char) == scr or scr in script_extensions(char)}
  characters = frozenset(characters)
  _DEFINED_CHARACTERS_CACHE[(version, scr)] = characters
  return characters


_strip_re = re.compile(r"[-'_ ]+")
def _folded_script_name(script_name):
  """Folds a script name to its bare bones for comparison."""
  # string.translate is changed by codecs, the method no longer takes two
  # parameters and so script_name.translate(None, "'-_ ") fails to compile
  return _strip_re.sub('', script_name)


def script_code(script_name):
  """Returns the four-letter ISO 15924 code of a script from its long name.
  """
  load_data()
  folded_script_name = _folded_script_name(script_name)
  try:
    return _HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE[folded_script_name]
  except:
    return _folded_script_name_to_code.get(folded_script_name, 'Zzzz')


_HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES = {
    'Aran': 'Nastaliq', # not assigned
    'Nkoo': 'N\'Ko',
    'Phag': 'Phags-Pa',
    'Piqd': 'Klingon', # not assigned
    'Zmth': 'Math', # not assigned
    'Zsye': 'Emoji', # not assigned
    'Zsym': 'Symbols', # not assigned
}

_HARD_CODED_FOLDED_SCRIPT_NAME_TO_CODE = {
    _folded_script_name(name): code for code, name in
    _HARD_CODED_HUMAN_READABLE_SCRIPT_NAMES.iteritems()
}

{'fraction_non_alphanumeric': 0.0902122641509434, 'fraction_numerical': 0.003900580551523948, 'mean_word_length': 4.28626228722129, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1847022', 'n_tokens_mistral': 7363, 'n_tokens_neox': 7151, 'n_words': 2196}