<commit_before># frozen_string_literal: true

require "sanscript/refinements"
require "sanscript/transliterate/schemes"
#
# Sanscript
#
# Sanscript is a Sanskrit transliteration library. Currently, it supports
# other Indian languages only incidentally.
#
# Released under the MIT and GPL Licenses.
#
module Sanscript
  using Refinements
  module Transliterate
    class << self
      attr_reader :defaults, :schemes, :scheme_names, :brahmic_schemes, :roman_schemes,
                  :all_alternates
    end

    @defaults = {
      skip_sgml: false,
      syncope: false,
    }

    @cache = {}

    module_function

    #
    #  Check whether the given scheme encodes Brahmic Sanskrit.
    #
    #  @param name  the scheme name
    #  @return      boolean
    #
    def brahmic_scheme?(name)
      @brahmic_schemes.include?(name.to_sym)
    end

    #
    #  Check whether the given scheme encodes romanized Sanskrit.
    #
    #  @param name  the scheme name
    #  @return      boolean
    #
    def roman_scheme?(name)
      @roman_schemes.include?(name.to_sym)
    end

    #
    # Add a Brahmic scheme to Sanscript.
    #
    # Schemes are of two types: "Brahmic" and "roman". Brahmic consonants
    # have an inherent vowel sound, but roman consonants do not. This is the
    # main difference between these two types of scheme.
    #
    # A scheme definition is an object ("{}") that maps a group name to a
    # list of characters. For illustration, see the "devanagari" scheme at
    # the top of this file.
    #
    # You can use whatever group names you like, but for the best results,
    # you should use the same group names that Sanscript does.
    #
    # @param name    the scheme name
    # @param scheme  the scheme data itself. This should be constructed as
    #                described above.
    #
    def add_brahmic_scheme(name, scheme)
      name = name.to_sym
      scheme = scheme.deep_dup
      @schemes[name] = scheme.deep_freeze
      @brahmic_schemes.add(name)
      @scheme_names.add(name)
      scheme
    end

    #
    # Add a roman scheme to Sanscript.
    #
    # See the comments on Sanscript.add_brahmic_scheme. The "vowel_marks" field
    # can be omitted.
    #
    # @param name    the scheme name
    # @param scheme  the scheme data itself
    #
    def add_roman_scheme(name, scheme)
      name = name.to_sym
      scheme = scheme.deep_dup
      scheme[:vowel_marks] = scheme[:vowels][1..-1] unless scheme.key?(:vowel_marks)
      @schemes[name] = scheme.deep_freeze
      @roman_schemes.add(name)
      @scheme_names.add(name)
      scheme
    end

    #
    # Create a deep copy of an object, for certain kinds of objects.
    #
    # @param scheme  the scheme to copy
    # @return        the copy
    #

    # Set up various schemes
    begin
      # Re-add existing Brahmic schemes in order to add them to `scheme_names`
      # and to freeze them up.
      brahmic_scheme_names = %i[bengali devanagari gujarati gurmukhi kannada malayalam
                                oriya tamil telugu]
      brahmic_scheme_names.each do |name|
        add_brahmic_scheme(name, @schemes[name])
      end

      # Set up roman schemes
      kolkata = @schemes[:kolkata] = @schemes[:iast].deep_dup
      roman_scheme_names = %i[iast itrans hk kolkata slp1 velthuis wx]
      kolkata[:vowels] = %w[a ā i ī u ū ṛ ṝ ḷ ḹ e ē ai o ō au]

      # These schemes already belong to Sanscript.schemes. But by adding
      # them again with `add_roman_scheme`, we automatically build up
      # `roman_schemes` and define a `vowel_marks` field for each one.
      roman_scheme_names.each do |name|
        add_roman_scheme(name, @schemes[name])
      end

      # ITRANS variant, which supports Dravidian short 'e' and 'o'.
      itrans_dravidian = @schemes[:itrans].deep_dup
      itrans_dravidian[:vowels] = %w[a A i I u U Ri RRI LLi LLi e E ai o O au]
      itrans_dravidian[:vowel_marks] = itrans_dravidian[:vowels][1..-1]
      @all_alternates[:itrans_dravidian] = @all_alternates[:itrans]
      add_roman_scheme(:itrans_dravidian, itrans_dravidian)

      # ensure deep freeze on alternates
      @all_alternates.each { |_, scheme| scheme.deep_freeze }
    end

    # /**
    # Transliterate from one script to another.
    #  *
    # @param data     the string to transliterate
    # @param from     the source script
    # @param to       the destination script
    # @param options  transliteration options
    # @return         the finished string
    #
    def transliterate(data, from, to, options = {})
      from = from.to_sym
      to = to.to_sym
      raise "Scheme not known ':#{from}'" unless @schemes.key?(from)
      raise "Scheme not known ':#{to}'" unless @schemes.key?(to)
<commit_msg>Add :kh copy of :hk scheme.<commit_after># frozen_string_literal: true

require "sanscript/refinements"
require "sanscript/transliterate/schemes"
#
# Sanscript
#
# Sanscript is a Sanskrit transliteration library. Currently, it supports
# other Indian languages only incidentally.
#
# Released under the MIT and GPL Licenses.
#
module Sanscript
  using Refinements
  module Transliterate
    class << self
      attr_reader :defaults, :schemes, :scheme_names, :brahmic_schemes, :roman_schemes,
                  :all_alternates
    end

    @defaults = {
      skip_sgml: false,
      syncope: false,
    }

    @cache = {}

    module_function

    #
    #  Check whether the given scheme encodes Brahmic Sanskrit.
    #
    #  @param name  the scheme name
    #  @return      boolean
    #
    def brahmic_scheme?(name)
      @brahmic_schemes.include?(name.to_sym)
    end

    #
    #  Check whether the given scheme encodes romanized Sanskrit.
    #
    #  @param name  the scheme name
    #  @return      boolean
    #
    def roman_scheme?(name)
      @roman_schemes.include?(name.to_sym)
    end

    #
    # Add a Brahmic scheme to Sanscript.
    #
    # Schemes are of two types: "Brahmic" and "roman". Brahmic consonants
    # have an inherent vowel sound, but roman consonants do not. This is the
    # main difference between these two types of scheme.
    #
    # A scheme definition is an object ("{}") that maps a group name to a
    # list of characters. For illustration, see the "devanagari" scheme at
    # the top of this file.
    #
    # You can use whatever group names you like, but for the best results,
    # you should use the same group names that Sanscript does.
    #
    # @param name    the scheme name
    # @param scheme  the scheme data itself. This should be constructed as
    #                described above.
    #
    def add_brahmic_scheme(name, scheme)
      name = name.to_sym
      scheme = scheme.deep_dup
      @schemes[name] = scheme.deep_freeze
      @brahmic_schemes.add(name)
      @scheme_names.add(name)
      scheme
    end

    #
    # Add a roman scheme to Sanscript.
    #
    # See the comments on Sanscript.add_brahmic_scheme. The "vowel_marks" field
    # can be omitted.
    #
    # @param name    the scheme name
    # @param scheme  the scheme data itself
    #
    def add_roman_scheme(name, scheme)
      name = name.to_sym
      scheme = scheme.deep_dup
      scheme[:vowel_marks] = scheme[:vowels][1..-1] unless scheme.key?(:vowel_marks)
      @schemes[name] = scheme.deep_freeze
      @roman_schemes.add(name)
      @scheme_names.add(name)
      scheme
    end

    #
    # Create a deep copy of an object, for certain kinds of objects.
    #
    # @param scheme  the scheme to copy
    # @return        the copy
    #

    # Set up various schemes
    begin
      # Re-add existing Brahmic schemes in order to add them to `scheme_names`
      # and to freeze them up.
      brahmic_scheme_names = %i[bengali devanagari gujarati gurmukhi kannada malayalam
                                oriya tamil telugu]
      brahmic_scheme_names.each do |name|
        add_brahmic_scheme(name, @schemes[name])
      end

      # Set up roman schemes
      kolkata = @schemes[:kolkata] = @schemes[:iast].deep_dup
      roman_scheme_names = %i[iast itrans hk kolkata slp1 velthuis wx]
      kolkata[:vowels] = %w[a ā i ī u ū ṛ ṝ ḷ ḹ e ē ai o ō au]

      # These schemes already belong to Sanscript.schemes. But by adding
      # them again with `add_roman_scheme`, we automatically build up
      # `roman_schemes` and define a `vowel_marks` field for each one.
      roman_scheme_names.each do |name|
        add_roman_scheme(name, @schemes[name])
      end

      # Add Kyoto-Harvard copy (alternate name of Harvard-Kyoto scheme)/
      add_roman_scheme(:kh, @schemes[:hk])

      # ITRANS variant, which supports Dravidian short 'e' and 'o'.
      itrans_dravidian = @schemes[:itrans].deep_dup
      itrans_dravidian[:vowels] = %w[a A i I u U Ri RRI LLi LLi e E ai o O au]
      itrans_dravidian[:vowel_marks] = itrans_dravidian[:vowels][1..-1]
      @all_alternates[:itrans_dravidian] = @all_alternates[:itrans]
      add_roman_scheme(:itrans_dravidian, itrans_dravidian)

      # ensure deep freeze on alternates
      @all_alternates.each { |_, scheme| scheme.deep_freeze }
    end

    # /**
    # Transliterate from one script to another.
    #  *
    # @param data     the string to transliterate
    # @param from     the source script
    # @param to       the destination script
    # @param options  transliteration options
    # @return         the finished string
    #
    def transliterate(data, from, to, options = {})
      from = from.to_sym
      to = to.to_sym
      raise "Scheme not known ':#{from}'" unless @schemes.key?(from)
      raise "Scheme not known ':#{to}'" unless @schemes.key?(to)

{'fraction_non_alphanumeric': 0.08313390906253251, 'fraction_numerical': 0.0010404744563520965, 'mean_word_length': 2.5260454878943506, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197180', 'n_tokens_mistral': 3176, 'n_tokens_neox': 3100, 'n_words': 959}