<commit_before># encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

require 'nokogiri'
require 'java'

require 'twitter_cldr/resources/download'

module TwitterCldr
  module Resources
    # This class should be used with JRuby 1.7 in 1.9 mode, ICU4J version >= 49.1,
    # and CLDR version <= 23 (v24 syntax is not supported yet).
    #
    class TailoringImporter < IcuBasedImporter

      SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
      SIMPLE_RULES      = %w[p s t i]
      LEVEL_RULE_REGEXP = /^(p|s|t|i)(c?)$/

      IGNORED_TAGS = %w[reset text #comment]

      LAST_BYTE_MASK = 0xFF

      LOCALES_MAP = {
        :'zh-Hant' => :'zh_Hant',
        id: :root,
        it: :root,
        ms: :root,
        nl: :root,
        pt: :root
      }

      EMPTY_TAILORING_DATA = {
        collator_options: {},
        tailored_table: '',
        suppressed_contractions: ''
      }

      class ImportError < RuntimeError; end

      # Arguments:
      #
      #   input_path  - path to a directory containing CLDR data
      #   output_path - output directory for imported YAML files
      #   icu4j_path  - path to ICU4J jar file
      #
      def initialize(input_path, output_path, icu4j_path)
        require_icu4j(icu4j_path)

        @input_path  = input_path
        @output_path = output_path
      end

      def import(locales)
        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
        locales.each { |locale| import_locale(locale) }
      end

      private<commit_msg>Reverting another hash to old syntax (for aesthetics)<commit_after># encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

require 'nokogiri'
require 'java'

require 'twitter_cldr/resources/download'

module TwitterCldr
  module Resources
    # This class should be used with JRuby 1.7 in 1.9 mode, ICU4J version >= 49.1,
    # and CLDR version <= 23 (v24 syntax is not supported yet).
    #
    class TailoringImporter < IcuBasedImporter

      SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
      SIMPLE_RULES      = %w[p s t i]
      LEVEL_RULE_REGEXP = /^(p|s|t|i)(c?)$/

      IGNORED_TAGS = %w[reset text #comment]

      LAST_BYTE_MASK = 0xFF

      LOCALES_MAP = {
        :'zh-Hant' => :'zh_Hant',
        :id => :root,
        :it => :root,
        :ms => :root,
        :nl => :root,
        :pt => :root
      }

      EMPTY_TAILORING_DATA = {
        collator_options: {},
        tailored_table: '',
        suppressed_contractions: ''
      }

      class ImportError < RuntimeError; end

      # Arguments:
      #
      #   input_path  - path to a directory containing CLDR data
      #   output_path - output directory for imported YAML files
      #   icu4j_path  - path to ICU4J jar file
      #
      def initialize(input_path, output_path, icu4j_path)
        require_icu4j(icu4j_path)

        @input_path  = input_path
        @output_path = output_path
      end

      def import(locales)
        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
        locales.each { |locale| import_locale(locale) }
      end

      private
{'fraction_non_alphanumeric': 0.09705042816365367, 'fraction_numerical': 0.01585791309863622, 'mean_word_length': 2.281997918834547, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5893337', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1063, 'n_words': 264}