<commit_before># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Some utilities to identify Noto fonts and collect them into families"""

import argparse
import collections
import os
from os import path
import re
import sys

from fontTools import ttLib

from nototools import cldr_data
from nototools import coverage
from nototools import font_data
from nototools import lang_data
from nototools import notoconfig
from nototools import noto_data
from nototools import tool_utils
from nototools import unicode_data

# The '[xxx]' syntax is used to get the noto-xxx value from notoconfig.
# for now we exclude alpha, the phase 3 fonts are here but we don't use
# them yet.
NOTO_FONT_PATHS = [
    '[fonts]/hinted', '[fonts]/unhinted', '[emoji]/fonts', '[cjk]']


ODD_SCRIPTS = {
  'CJKjp': 'Jpan',
  'CJKkr': 'Kore',
  'CJKsc': 'Hans',
  'CJKtc': 'Hant',
  'JP': 'Jpan',
  'KR': 'Kore',
  'SC': 'Hans',
  'TC': 'Hant',
  'NKo': 'Nkoo',
  'SumeroAkkadianCuneiform': 'Xsux',
  'Symbols': 'Zsym',
  'Emoji': 'Zsye',
}


def convert_to_four_letter(script_name):
  """Converts a script name from a Noto font file name to ISO 15924 code."""
  if not script_name:
    raise ValueError('empty script name')
  if script_name in ODD_SCRIPTS:
    return ODD_SCRIPTS[script_name]
  script_code = unicode_data.script_code(script_name)
  if script_code == 'Zzzz':
    if len(script_name) != 4:
      raise ValueError('no script for %s' % script_name)
    print >> sys.stderr, 'defaulting script for %s' % script_name
    script_code = script_name
  return script_code


def preferred_script_name(script_key):
  try:
    return unicode_data.human_readable_script_name(script_key)
  except:
    return cldr_data.get_english_script_name(script_key)


_script_key_to_report_name = {
    'Aran': '(Urdu)',  # phase 2 usage
    'HST': '(Historic)',
    'LGC': '(LGC)',
    'SYM2': 'Symbols2'
}
def script_name_for_report(script_key):
    return (_script_key_to_report_name.get(script_key, None) or
            preferred_script_name(script_key))


# NotoFont maps a font path to information we assume the font to have, based
# on Noto path and naming conventions:
# - filepath: the path name from which we derived the information
# - family: family name, e.g. 'Arimo', 'Noto'
# - style: type style, e.g. 'Sans', 'Serif', might be None
# - script: four-letter script code or 'private use' code like 'Aran', 'LGC',
#     'HST'
# - variant: script variant like 'UI' or Syriac variants like 'Estrangela'
# - width: width name ('Condensed') or None
# - weight: weight name
# - slope: slope name ('Italic') or None
# - fmt: 'ttf', 'otf', or 'otc'
# - manufacturer: 'Adobe', 'Google', 'Khmertype', or 'Monotype'
# - license_type: 'sil' or 'apache'
# - is_hinted: boolean, true if hinted
# - is_mono: boolean, true if monospace (currently CJK Latin range, or legacy
#     LGC Mono)
# - is_display: boolean, true if display
# - is_UI: boolean, true if has UI in the name
# - is_UI_metrics: boolean true if must have UI metrics
# - is_cjk: boolean, true if a CJK font (from Adobe)
# - subset: name of cjk subset (KR, JA, SC, TC) for reduced-charset fonts
#     targeted at these languages
NotoFont = collections.namedtuple(
    'NotoFont',
    'filepath, family, style, script, variant, width, weight, slope, '
    'fmt, manufacturer, license_type, is_hinted, is_mono, is_UI, is_UI_metrics, '
    'is_display, is_cjk, subset')


# These are the ideal pseudo-css weights. 'ideal', because windows GDI limits us
# to weights >= 250 if we are to prevent auto-bolding, and 'pseudo-css' because
# css limits us to multiples of 100 currently.  The hope is that both of these
# restrictions eventually go away, so we encode the values as we wish they would
# be, and adjust when necessary based on context.
WEIGHTS = {
    'Thin': 100,
    'ExtraLight': 200,
    'Light': 300,
    'DemiLight': 350,
    'Regular': 400,
    'Medium': 500,
    'SemiBold': 600,
    'Bold': 700,
    'ExtraBold': 800,
    'Black': 900
}

# Currently we only have one name per weight.  Eventually this can break.
WEIGHT_TO_STRING = {s:w for w, s in WEIGHTS.iteritems()}


_FONT_NAME_REGEX = (
    # family should be prepended - this is so Roboto can be used with unittests
    # that use this regex to parse.
    '(Sans|Serif|Naskh|Kufi|Nastaliq|Emoji|ColorEmoji)?'
    '(Mono(?:space)?)?'
    '(.*?)'
    '(Eastern|Estrangela|Western|Slanted|New|Unjoined)?'
    '(UI)?'
    '(Display)?'
    '-?'
    '((?:Semi|Extra)?Condensed)?'
    '(|%s)?' % '|'.join(WEIGHTS.keys()) +
    '(Italic)?'
    '\.(ttf|ttc|otf)')


_EXT_REGEX = re.compile(r'.*\.(?:ttf|ttc|otf)$')

def get_noto_font(filepath, family_name='Arimo|Cousine|Tinos|Noto',
                  phase=3):
  """Return a NotoFont if filepath points to a noto font, or None if we can't
  process the path."""

  filedir, filename = os.path.split(filepath)
  if not filedir:
    filedir = os.getcwd()
  match = match_filename(filename, family_name)
  if match:<commit_msg>Support both DemiLight and SemiLight names.<commit_after># Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Some utilities to identify Noto fonts and collect them into families"""

import argparse
import collections
import os
from os import path
import re
import sys

from fontTools import ttLib

from nototools import cldr_data
from nototools import coverage
from nototools import font_data
from nototools import lang_data
from nototools import notoconfig
from nototools import noto_data
from nototools import tool_utils
from nototools import unicode_data

# The '[xxx]' syntax is used to get the noto-xxx value from notoconfig.
# for now we exclude alpha, the phase 3 fonts are here but we don't use
# them yet.
NOTO_FONT_PATHS = [
    '[fonts]/hinted', '[fonts]/unhinted', '[emoji]/fonts', '[cjk]']


ODD_SCRIPTS = {
  'CJKjp': 'Jpan',
  'CJKkr': 'Kore',
  'CJKsc': 'Hans',
  'CJKtc': 'Hant',
  'JP': 'Jpan',
  'KR': 'Kore',
  'SC': 'Hans',
  'TC': 'Hant',
  'NKo': 'Nkoo',
  'SumeroAkkadianCuneiform': 'Xsux',
  'Symbols': 'Zsym',
  'Emoji': 'Zsye',
}


def convert_to_four_letter(script_name):
  """Converts a script name from a Noto font file name to ISO 15924 code."""
  if not script_name:
    raise ValueError('empty script name')
  if script_name in ODD_SCRIPTS:
    return ODD_SCRIPTS[script_name]
  script_code = unicode_data.script_code(script_name)
  if script_code == 'Zzzz':
    if len(script_name) != 4:
      raise ValueError('no script for %s' % script_name)
    print >> sys.stderr, 'defaulting script for %s' % script_name
    script_code = script_name
  return script_code


def preferred_script_name(script_key):
  try:
    return unicode_data.human_readable_script_name(script_key)
  except:
    return cldr_data.get_english_script_name(script_key)


_script_key_to_report_name = {
    'Aran': '(Urdu)',  # phase 2 usage
    'HST': '(Historic)',
    'LGC': '(LGC)',
    'SYM2': 'Symbols2'
}
def script_name_for_report(script_key):
    return (_script_key_to_report_name.get(script_key, None) or
            preferred_script_name(script_key))


# NotoFont maps a font path to information we assume the font to have, based
# on Noto path and naming conventions:
# - filepath: the path name from which we derived the information
# - family: family name, e.g. 'Arimo', 'Noto'
# - style: type style, e.g. 'Sans', 'Serif', might be None
# - script: four-letter script code or 'private use' code like 'Aran', 'LGC',
#     'HST'
# - variant: script variant like 'UI' or Syriac variants like 'Estrangela'
# - width: width name ('Condensed') or None
# - weight: weight name
# - slope: slope name ('Italic') or None
# - fmt: 'ttf', 'otf', or 'otc'
# - manufacturer: 'Adobe', 'Google', 'Khmertype', or 'Monotype'
# - license_type: 'sil' or 'apache'
# - is_hinted: boolean, true if hinted
# - is_mono: boolean, true if monospace (currently CJK Latin range, or legacy
#     LGC Mono)
# - is_display: boolean, true if display
# - is_UI: boolean, true if has UI in the name
# - is_UI_metrics: boolean true if must have UI metrics
# - is_cjk: boolean, true if a CJK font (from Adobe)
# - subset: name of cjk subset (KR, JA, SC, TC) for reduced-charset fonts
#     targeted at these languages
NotoFont = collections.namedtuple(
    'NotoFont',
    'filepath, family, style, script, variant, width, weight, slope, '
    'fmt, manufacturer, license_type, is_hinted, is_mono, is_UI, is_UI_metrics, '
    'is_display, is_cjk, subset')


# These are the ideal pseudo-css weights. 'ideal', because windows GDI limits us
# to weights >= 250 if we are to prevent auto-bolding, and 'pseudo-css' because
# css limits us to multiples of 100 currently.  The hope is that both of these
# restrictions eventually go away, so we encode the values as we wish they would
# be, and adjust when necessary based on context.
WEIGHTS = {
    'Thin': 100,
    'ExtraLight': 200,
    'Light': 300,
    'DemiLight': 350,  # used in cjk fonts
    'SemiLight': 350,  # used in other noto fonts
    'Regular': 400,
    'Medium': 500,
    'SemiBold': 600,
    'Bold': 700,
    'ExtraBold': 800,
    'Black': 900
}

_FONT_NAME_REGEX = (
    # family should be prepended - this is so Roboto can be used with unittests
    # that use this regex to parse.
    '(Sans|Serif|Naskh|Kufi|Nastaliq|Emoji|ColorEmoji)?'
    '(Mono(?:space)?)?'
    '(.*?)'
    '(Eastern|Estrangela|Western|Slanted|New|Unjoined)?'
    '(UI)?'
    '(Display)?'
    '-?'
    '((?:Semi|Extra)?Condensed)?'
    '(|%s)?' % '|'.join(WEIGHTS.keys()) +
    '(Italic)?'
    '\.(ttf|ttc|otf)')


_EXT_REGEX = re.compile(r'.*\.(?:ttf|ttc|otf)$')

def get_noto_font(filepath, family_name='Arimo|Cousine|Tinos|Noto',
                  phase=3):
  """Return a NotoFont if filepath points to a noto font, or None if we can't
  process the path."""

  filedir, filename = os.path.split(filepath)
  if not filedir:
    filedir = os.getcwd()
  match = match_filename(filename, family_name)
  if match:
{'fraction_non_alphanumeric': 0.11990538573508006, 'fraction_numerical': 0.010280203784570598, 'mean_word_length': 3.9674649796656123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675463', 'n_tokens_mistral': 3918, 'n_tokens_neox': 3713, 'n_words': 1272}