<commit_before># -*- encoding: utf-8 -*-

require "unicode_utils/canonical_decomposition"
require "unicode_utils/combining_class"
require "unicode_utils/read_codepoint_set"

module UnicodeUtils
  
  COMPOSITION_EXCLUSION_SET = # :nodoc:
    Impl.read_codepoint_set("composition_exclusion_set")

  CANONICAL_COMPOSITION_MAP = Hash.new.tap do |m|
    CANONICAL_DECOMPOSITION_MAP.each_pair { |comp, decomp|
      if decomp.length == 2
        (m[decomp[0]] ||= {})[decomp[1]] = comp
      end
    }
  end

  module Impl
    
    module NFC

      def self.starter?(cp)
        (COMBINING_CLASS_MAP[cp] || 0) == 0
      end

      def self.nonstarter?(cp)
        !starter(cp)
      end

      # does b block c?
      def self.blocked?(b, c)
        # From the standard:
        # "If a combining character sequence is in canonical order,
        # then testing whether a character is blocked requires looking
        # at only the immediately preceding character."
        # cpary is in canonical order (since it comes out of
        # canonical_decomposition).
        (COMBINING_CLASS_MAP[b] || 0) >= (COMBINING_CLASS_MAP[c] || 0)
      end

      def self.primary_composite?(cp)
        unless CANONICAL_DECOMPOSITION_MAP[cp] ||
            # has hangul syllable decomposition?
            (cp >= 0xAC00 && cp <= 0xD7A3)
          return false
        end
        !COMPOSITION_EXCLUSION_SET.include?(cp)
      end<commit_msg>nfc.rb: remove unused nonstarter? method<commit_after># -*- encoding: utf-8 -*-

require "unicode_utils/canonical_decomposition"
require "unicode_utils/combining_class"
require "unicode_utils/read_codepoint_set"

module UnicodeUtils
  
  COMPOSITION_EXCLUSION_SET = # :nodoc:
    Impl.read_codepoint_set("composition_exclusion_set")

  CANONICAL_COMPOSITION_MAP = Hash.new.tap do |m|
    CANONICAL_DECOMPOSITION_MAP.each_pair { |comp, decomp|
      if decomp.length == 2
        (m[decomp[0]] ||= {})[decomp[1]] = comp
      end
    }
  end

  module Impl
    
    module NFC

      def self.starter?(cp)
        (COMBINING_CLASS_MAP[cp] || 0) == 0
      end

      # does b block c?
      def self.blocked?(b, c)
        # From the standard:
        # "If a combining character sequence is in canonical order,
        # then testing whether a character is blocked requires looking
        # at only the immediately preceding character."
        # cpary is in canonical order (since it comes out of
        # canonical_decomposition).
        (COMBINING_CLASS_MAP[b] || 0) >= (COMBINING_CLASS_MAP[c] || 0)
      end

      def self.primary_composite?(cp)
        unless CANONICAL_DECOMPOSITION_MAP[cp] ||
            # has hangul syllable decomposition?
            (cp >= 0xAC00 && cp <= 0xD7A3)
          return false
        end
        !COMPOSITION_EXCLUSION_SET.include?(cp)
      end
{'fraction_non_alphanumeric': 0.09594882729211088, 'fraction_numerical': 0.009950248756218905, 'mean_word_length': 2.608974358974359, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '84928', 'n_tokens_mistral': 949, 'n_tokens_neox': 887, 'n_words': 219}