<commit_before># encoding: utf-8
require 'cgi'
require 'rexml/document'

module Twine
  module Formatters
    class Android < Abstract
      include Twine::Placeholders

      def format_name
        'android'
      end

      def extension
        '.xml'
      end

      def can_handle_directory?(path)
        Dir.entries(path).any? { |item| /^values.*$/.match(item) }
      end

      def default_file_name
        'strings.xml'
      end

      def determine_language_given_path(path)
        path_arr = path.split(File::SEPARATOR)
        path_arr.each do |segment|
          if segment == 'values'
            return @twine_file.language_codes[0]
          else
            # The language is defined by a two-letter ISO 639-1 language code, optionally followed by a two letter ISO 3166-1-alpha-2 region code (preceded by lowercase "r").
            # see http://developer.android.com/guide/topics/resources/providing-resources.html#AlternativeResources
            match = /^values-([a-z]{2}(-r[a-z]{2})?)$/i.match(segment)
            
            return match[1].sub('-r', '-') if match
          end
        end

        return
      end

      def output_path_for_language(lang)
        "values-#{lang}"
      end

      def set_translation_for_key(key, lang, value)
        value = CGI.unescapeHTML(value)
        value.gsub!('\\\'', '\'')
        value.gsub!('\\"', '"')
        value = convert_placeholders_from_android_to_twine(value)
        value.gsub!('\@', '@')
        value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
        super(key, lang, value)
      end

      def read(io, lang)
        document = REXML::Document.new io, :compress_whitespace => %w{ string }

        comment = nil
        document.root.children.each do |child|
          if child.is_a? REXML::Comment
            content = child.string.strip
            comment = content if content.length > 0 and not content.start_with?("SECTION:")
          elsif child.is_a? REXML::Element
            next unless child.name == 'string'

            key = child.attributes['name']

            set_translation_for_key(key, lang, child.text)
            set_comment_for_key(key, comment) if comment

            comment = nil
          end 
        end<commit_msg>Android folder should have the region code preceded by lowercase "r"<commit_after># encoding: utf-8
require 'cgi'
require 'rexml/document'

module Twine
  module Formatters
    class Android < Abstract
      include Twine::Placeholders

      def format_name
        'android'
      end

      def extension
        '.xml'
      end

      def can_handle_directory?(path)
        Dir.entries(path).any? { |item| /^values.*$/.match(item) }
      end

      def default_file_name
        'strings.xml'
      end

      def determine_language_given_path(path)
        path_arr = path.split(File::SEPARATOR)
        path_arr.each do |segment|
          if segment == 'values'
            return @twine_file.language_codes[0]
          else
            # The language is defined by a two-letter ISO 639-1 language code, optionally followed by a two letter ISO 3166-1-alpha-2 region code (preceded by lowercase "r").
            # see http://developer.android.com/guide/topics/resources/providing-resources.html#AlternativeResources
            match = /^values-([a-z]{2}(-r[a-z]{2})?)$/i.match(segment)

            return match[1].sub('-r', '-') if match
          end
        end

        return
      end

      def output_path_for_language(lang)
          "values-#{lang}".gsub(/-(\p{Lu})/, '-r\1')
      end

      def set_translation_for_key(key, lang, value)
        value = CGI.unescapeHTML(value)
        value.gsub!('\\\'', '\'')
        value.gsub!('\\"', '"')
        value = convert_placeholders_from_android_to_twine(value)
        value.gsub!('\@', '@')
        value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
        super(key, lang, value)
      end

      def read(io, lang)
        document = REXML::Document.new io, :compress_whitespace => %w{ string }

        comment = nil
        document.root.children.each do |child|
          if child.is_a? REXML::Comment
            content = child.string.strip
            comment = content if content.length > 0 and not content.start_with?("SECTION:")
          elsif child.is_a? REXML::Element
            next unless child.name == 'string'

            key = child.attributes['name']

            set_translation_for_key(key, lang, child.text)
            set_comment_for_key(key, comment) if comment

            comment = nil
          end 
        end
{'fraction_non_alphanumeric': 0.12383632820956918, 'fraction_numerical': 0.011041350941762286, 'mean_word_length': 2.057577763070814, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112650', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1394, 'n_words': 318}